{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "10e99e49-854e-461c-b21d-678feedf31e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import findspark\n",
    "from pyspark.sql import SparkSession\n",
    "import pyspark.sql.functions as F\n",
    "import pyspark.sql.types as T\n",
    "import pandas as pd\n",
    "from functools import reduce\n",
    "\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0aa19bc7-dfd5-405c-ba20-2b2d0c67819d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "Exception occurred during processing of request from ('127.0.0.1', 49292)\n",
      "Traceback (most recent call last):\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\socketserver.py\", line 316, in _handle_request_noblock\n",
      "    self.process_request(request, client_address)\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\socketserver.py\", line 347, in process_request\n",
      "    self.finish_request(request, client_address)\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\socketserver.py\", line 360, in finish_request\n",
      "    self.RequestHandlerClass(request, client_address, self)\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\socketserver.py\", line 747, in __init__\n",
      "    self.handle()\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\site-packages\\pyspark\\accumulators.py\", line 295, in handle\n",
      "    poll(accum_updates)\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\site-packages\\pyspark\\accumulators.py\", line 267, in poll\n",
      "    if self.rfile in r and func():\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\site-packages\\pyspark\\accumulators.py\", line 271, in accum_updates\n",
      "    num_updates = read_int(self.rfile)\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\site-packages\\pyspark\\serializers.py\", line 594, in read_int\n",
      "    length = stream.read(4)\n",
      "  File \"D:\\AppData\\Local\\mambaforge\\envs\\python39ml\\lib\\socket.py\", line 704, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] 远程主机强迫关闭了一个现有的连接。\n",
      "----------------------------------------\n"
     ]
    }
   ],
   "source": [
    "spark = SparkSession.builder.master(\"local[*]\").appName(\"Recipes ML model\").config(\"spark.driver.memory\",\"8g\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "42e37b27-9a78-429f-ae33-b7743effd158",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20057 680\n",
      "root\n",
      " |-- title: string (nullable = true)\n",
      " |-- rating: string (nullable = true)\n",
      " |-- calories: string (nullable = true)\n",
      " |-- protein: double (nullable = true)\n",
      " |-- fat: double (nullable = true)\n",
      " |-- sodium: double (nullable = true)\n",
      " |-- #cakeweek: double (nullable = true)\n",
      " |-- #wasteless: double (nullable = true)\n",
      " |-- 22-minute meals: double (nullable = true)\n",
      " |-- 3-ingredient recipes: double (nullable = true)\n",
      " |-- 30 days of groceries: double (nullable = true)\n",
      " |-- advance prep required: double (nullable = true)\n",
      " |-- alabama: double (nullable = true)\n",
      " |-- alaska: double (nullable = true)\n",
      " |-- alcoholic: double (nullable = true)\n",
      " |-- almond: double (nullable = true)\n",
      " |-- amaretto: double (nullable = true)\n",
      " |-- anchovy: double (nullable = true)\n",
      " |-- anise: double (nullable = true)\n",
      " |-- anniversary: double (nullable = true)\n",
      " |-- anthony bourdain: double (nullable = true)\n",
      " |-- aperitif: double (nullable = true)\n",
      " |-- appetizer: double (nullable = true)\n",
      " |-- apple: double (nullable = true)\n",
      " |-- apple juice: double (nullable = true)\n",
      " |-- apricot: double (nullable = true)\n",
      " |-- arizona: double (nullable = true)\n",
      " |-- artichoke: double (nullable = true)\n",
      " |-- arugula: double (nullable = true)\n",
      " |-- asian pear: double (nullable = true)\n",
      " |-- asparagus: double (nullable = true)\n",
      " |-- aspen: double (nullable = true)\n",
      " |-- atlanta: double (nullable = true)\n",
      " |-- australia: double (nullable = true)\n",
      " |-- avocado: double (nullable = true)\n",
      " |-- back to school: double (nullable = true)\n",
      " |-- backyard bbq: double (nullable = true)\n",
      " |-- bacon: double (nullable = true)\n",
      " |-- bake: double (nullable = true)\n",
      " |-- banana: double (nullable = true)\n",
      " |-- barley: double (nullable = true)\n",
      " |-- basil: double (nullable = true)\n",
      " |-- bass: double (nullable = true)\n",
      " |-- bastille day: double (nullable = true)\n",
      " |-- bean: double (nullable = true)\n",
      " |-- beef: double (nullable = true)\n",
      " |-- beef rib: double (nullable = true)\n",
      " |-- beef shank: double (nullable = true)\n",
      " |-- beef tenderloin: double (nullable = true)\n",
      " |-- beer: double (nullable = true)\n",
      " |-- beet: double (nullable = true)\n",
      " |-- bell pepper: double (nullable = true)\n",
      " |-- berry: double (nullable = true)\n",
      " |-- beverly hills: double (nullable = true)\n",
      " |-- birthday: double (nullable = true)\n",
      " |-- biscuit: double (nullable = true)\n",
      " |-- bitters: double (nullable = true)\n",
      " |-- blackberry: double (nullable = true)\n",
      " |-- blender: double (nullable = true)\n",
      " |-- blue cheese: double (nullable = true)\n",
      " |-- blueberry: double (nullable = true)\n",
      " |-- boil: double (nullable = true)\n",
      " |-- bok choy: double (nullable = true)\n",
      " |-- bon appétit: double (nullable = true)\n",
      " |-- bon app��tit: double (nullable = true)\n",
      " |-- boston: double (nullable = true)\n",
      " |-- bourbon: double (nullable = true)\n",
      " |-- braise: double (nullable = true)\n",
      " |-- bran: double (nullable = true)\n",
      " |-- brandy: double (nullable = true)\n",
      " |-- bread: double (nullable = true)\n",
      " |-- breadcrumbs: double (nullable = true)\n",
      " |-- breakfast: double (nullable = true)\n",
      " |-- brie: double (nullable = true)\n",
      " |-- brine: double (nullable = true)\n",
      " |-- brisket: double (nullable = true)\n",
      " |-- broccoli: double (nullable = true)\n",
      " |-- broccoli rabe: double (nullable = true)\n",
      " |-- broil: double (nullable = true)\n",
      " |-- brooklyn: double (nullable = true)\n",
      " |-- brown rice: double (nullable = true)\n",
      " |-- brownie: double (nullable = true)\n",
      " |-- brunch: double (nullable = true)\n",
      " |-- brussel sprout: double (nullable = true)\n",
      " |-- buffalo: double (nullable = true)\n",
      " |-- buffet: double (nullable = true)\n",
      " |-- bulgaria: double (nullable = true)\n",
      " |-- bulgur: double (nullable = true)\n",
      " |-- burrito: double (nullable = true)\n",
      " |-- butter: double (nullable = true)\n",
      " |-- buttermilk: double (nullable = true)\n",
      " |-- butternut squash: double (nullable = true)\n",
      " |-- butterscotch/caramel: double (nullable = true)\n",
      " |-- cabbage: double (nullable = true)\n",
      " |-- cake: double (nullable = true)\n",
      " |-- california: double (nullable = true)\n",
      " |-- calvados: double (nullable = true)\n",
      " |-- cambridge: double (nullable = true)\n",
      " |-- campari: double (nullable = true)\n",
      " |-- camping: double (nullable = true)\n",
      " |-- canada: double (nullable = true)\n",
      " |-- candy: double (nullable = true)\n",
      " |-- candy thermometer: double (nullable = true)\n",
      " |-- cantaloupe: double (nullable = true)\n",
      " |-- capers: double (nullable = true)\n",
      " |-- caraway: double (nullable = true)\n",
      " |-- cardamom: double (nullable = true)\n",
      " |-- carrot: double (nullable = true)\n",
      " |-- cashew: double (nullable = true)\n",
      " |-- casserole/gratin: double (nullable = true)\n",
      " |-- cauliflower: double (nullable = true)\n",
      " |-- caviar: double (nullable = true)\n",
      " |-- celery: double (nullable = true)\n",
      " |-- chambord: double (nullable = true)\n",
      " |-- champagne: double (nullable = true)\n",
      " |-- chard: double (nullable = true)\n",
      " |-- chartreuse: double (nullable = true)\n",
      " |-- cheddar: double (nullable = true)\n",
      " |-- cheese: double (nullable = true)\n",
      " |-- cherry: double (nullable = true)\n",
      " |-- chestnut: double (nullable = true)\n",
      " |-- chicago: double (nullable = true)\n",
      " |-- chicken: double (nullable = true)\n",
      " |-- chickpea: double (nullable = true)\n",
      " |-- chile: double (nullable = true)\n",
      " |-- chile pepper: double (nullable = true)\n",
      " |-- chili: double (nullable = true)\n",
      " |-- chill: double (nullable = true)\n",
      " |-- chive: double (nullable = true)\n",
      " |-- chocolate: double (nullable = true)\n",
      " |-- christmas: double (nullable = true)\n",
      " |-- christmas eve: double (nullable = true)\n",
      " |-- cilantro: double (nullable = true)\n",
      " |-- cinco de mayo: double (nullable = true)\n",
      " |-- cinnamon: double (nullable = true)\n",
      " |-- citrus: double (nullable = true)\n",
      " |-- clam: double (nullable = true)\n",
      " |-- clove: double (nullable = true)\n",
      " |-- cobbler/crumble: double (nullable = true)\n",
      " |-- cocktail: double (nullable = true)\n",
      " |-- cocktail party: double (nullable = true)\n",
      " |-- coconut: double (nullable = true)\n",
      " |-- cod: double (nullable = true)\n",
      " |-- coffee: double (nullable = true)\n",
      " |-- coffee grinder: double (nullable = true)\n",
      " |-- cognac/armagnac: double (nullable = true)\n",
      " |-- collard greens: double (nullable = true)\n",
      " |-- colorado: double (nullable = true)\n",
      " |-- columbus: double (nullable = true)\n",
      " |-- condiment: double (nullable = true)\n",
      " |-- condiment/spread: double (nullable = true)\n",
      " |-- connecticut: double (nullable = true)\n",
      " |-- cook like a diner: double (nullable = true)\n",
      " |-- cookbook critic: double (nullable = true)\n",
      " |-- cookie: double (nullable = true)\n",
      " |-- cookies: double (nullable = true)\n",
      " |-- coriander: double (nullable = true)\n",
      " |-- corn: double (nullable = true)\n",
      " |-- cornmeal: double (nullable = true)\n",
      " |-- costa mesa: double (nullable = true)\n",
      " |-- cottage cheese: double (nullable = true)\n",
      " |-- couscous: double (nullable = true)\n",
      " |-- crab: double (nullable = true)\n",
      " |-- cranberry: double (nullable = true)\n",
      " |-- cranberry sauce: double (nullable = true)\n",
      " |-- cream cheese: double (nullable = true)\n",
      " |-- créme de cacao: double (nullable = true)\n",
      " |-- crêpe: double (nullable = true)\n",
      " |-- cr��me de cacao: double (nullable = true)\n",
      " |-- cuba: double (nullable = true)\n",
      " |-- cucumber: double (nullable = true)\n",
      " |-- cumin: double (nullable = true)\n",
      " |-- cupcake: double (nullable = true)\n",
      " |-- currant: double (nullable = true)\n",
      " |-- curry: double (nullable = true)\n",
      " |-- custard: double (nullable = true)\n",
      " |-- dairy: double (nullable = true)\n",
      " |-- dairy free: double (nullable = true)\n",
      " |-- dallas: double (nullable = true)\n",
      " |-- date: double (nullable = true)\n",
      " |-- deep-fry: double (nullable = true)\n",
      " |-- denver: double (nullable = true)\n",
      " |-- dessert: double (nullable = true)\n",
      " |-- digestif: double (nullable = true)\n",
      " |-- dill: double (nullable = true)\n",
      " |-- dinner: double (nullable = true)\n",
      " |-- dip: double (nullable = true)\n",
      " |-- diwali: double (nullable = true)\n",
      " |-- dominican republic: double (nullable = true)\n",
      " |-- dorie greenspan: double (nullable = true)\n",
      " |-- double boiler: double (nullable = true)\n",
      " |-- dried fruit: double (nullable = true)\n",
      " |-- drink: double (nullable = true)\n",
      " |-- drinks: double (nullable = true)\n",
      " |-- duck: double (nullable = true)\n",
      " |-- easter: double (nullable = true)\n",
      " |-- eau de vie: double (nullable = true)\n",
      " |-- edible gift: double (nullable = true)\n",
      " |-- egg: double (nullable = true)\n",
      " |-- egg nog: double (nullable = true)\n",
      " |-- eggplant: double (nullable = true)\n",
      " |-- egypt: double (nullable = true)\n",
      " |-- emeril lagasse: double (nullable = true)\n",
      " |-- endive: double (nullable = true)\n",
      " |-- engagement party: double (nullable = true)\n",
      " |-- england: double (nullable = true)\n",
      " |-- entertaining: double (nullable = true)\n",
      " |-- epi + ushg: double (nullable = true)\n",
      " |-- epi loves the microwave: double (nullable = true)\n",
      " |-- escarole: double (nullable = true)\n",
      " |-- fall: double (nullable = true)\n",
      " |-- family reunion: double (nullable = true)\n",
      " |-- fat free: double (nullable = true)\n",
      " |-- father's day: double (nullable = true)\n",
      " |-- fennel: double (nullable = true)\n",
      " |-- feta: double (nullable = true)\n",
      " |-- fig: double (nullable = true)\n",
      " |-- fish: double (nullable = true)\n",
      " |-- flaming hot summer: double (nullable = true)\n",
      " |-- flat bread: double (nullable = true)\n",
      " |-- florida: double (nullable = true)\n",
      " |-- fontina: double (nullable = true)\n",
      " |-- food processor: double (nullable = true)\n",
      " |-- fortified wine: double (nullable = true)\n",
      " |-- fourth of july: double (nullable = true)\n",
      " |-- france: double (nullable = true)\n",
      " |-- frangelico: double (nullable = true)\n",
      " |-- frankenrecipe: double (nullable = true)\n",
      " |-- freeze/chill: double (nullable = true)\n",
      " |-- freezer food: double (nullable = true)\n",
      " |-- friendsgiving: double (nullable = true)\n",
      " |-- frittata: double (nullable = true)\n",
      " |-- fritter: double (nullable = true)\n",
      " |-- frozen dessert: double (nullable = true)\n",
      " |-- fruit: double (nullable = true)\n",
      " |-- fruit juice: double (nullable = true)\n",
      " |-- fry: double (nullable = true)\n",
      " |-- game: double (nullable = true)\n",
      " |-- garlic: double (nullable = true)\n",
      " |-- georgia: double (nullable = true)\n",
      " |-- germany: double (nullable = true)\n",
      " |-- gin: double (nullable = true)\n",
      " |-- ginger: double (nullable = true)\n",
      " |-- goat cheese: double (nullable = true)\n",
      " |-- goose: double (nullable = true)\n",
      " |-- gouda: double (nullable = true)\n",
      " |-- gourmet: double (nullable = true)\n",
      " |-- graduation: double (nullable = true)\n",
      " |-- grains: double (nullable = true)\n",
      " |-- grand marnier: double (nullable = true)\n",
      " |-- granola: double (nullable = true)\n",
      " |-- grape: double (nullable = true)\n",
      " |-- grapefruit: double (nullable = true)\n",
      " |-- grappa: double (nullable = true)\n",
      " |-- green bean: double (nullable = true)\n",
      " |-- green onion/scallion: double (nullable = true)\n",
      " |-- grill: double (nullable = true)\n",
      " |-- grill/barbecue: double (nullable = true)\n",
      " |-- ground beef: double (nullable = true)\n",
      " |-- ground lamb: double (nullable = true)\n",
      " |-- guam: double (nullable = true)\n",
      " |-- guava: double (nullable = true)\n",
      " |-- haiti: double (nullable = true)\n",
      " |-- halibut: double (nullable = true)\n",
      " |-- halloween: double (nullable = true)\n",
      " |-- ham: double (nullable = true)\n",
      " |-- hamburger: double (nullable = true)\n",
      " |-- hanukkah: double (nullable = true)\n",
      " |-- harpercollins: double (nullable = true)\n",
      " |-- hawaii: double (nullable = true)\n",
      " |-- hazelnut: double (nullable = true)\n",
      " |-- healdsburg: double (nullable = true)\n",
      " |-- healthy: double (nullable = true)\n",
      " |-- herb: double (nullable = true)\n",
      " |-- high fiber: double (nullable = true)\n",
      " |-- hollywood: double (nullable = true)\n",
      " |-- hominy/cornmeal/masa: double (nullable = true)\n",
      " |-- honey: double (nullable = true)\n",
      " |-- honeydew: double (nullable = true)\n",
      " |-- hors d'oeuvre: double (nullable = true)\n",
      " |-- horseradish: double (nullable = true)\n",
      " |-- hot drink: double (nullable = true)\n",
      " |-- hot pepper: double (nullable = true)\n",
      " |-- house & garden: double (nullable = true)\n",
      " |-- house cocktail: double (nullable = true)\n",
      " |-- houston: double (nullable = true)\n",
      " |-- hummus: double (nullable = true)\n",
      " |-- ice cream: double (nullable = true)\n",
      " |-- ice cream machine: double (nullable = true)\n",
      " |-- iced coffee: double (nullable = true)\n",
      " |-- iced tea: double (nullable = true)\n",
      " |-- idaho: double (nullable = true)\n",
      " |-- illinois: double (nullable = true)\n",
      " |-- indiana: double (nullable = true)\n",
      " |-- iowa: double (nullable = true)\n",
      " |-- ireland: double (nullable = true)\n",
      " |-- israel: double (nullable = true)\n",
      " |-- italy: double (nullable = true)\n",
      " |-- jalapeño: double (nullable = true)\n",
      " |-- jam or jelly: double (nullable = true)\n",
      " |-- jamaica: double (nullable = true)\n",
      " |-- japan: double (nullable = true)\n",
      " |-- jerusalem artichoke: double (nullable = true)\n",
      " |-- juicer: double (nullable = true)\n",
      " |-- jícama: double (nullable = true)\n",
      " |-- kahlúa: double (nullable = true)\n",
      " |-- kale: double (nullable = true)\n",
      " |-- kansas: double (nullable = true)\n",
      " |-- kansas city: double (nullable = true)\n",
      " |-- kentucky: double (nullable = true)\n",
      " |-- kentucky derby: double (nullable = true)\n",
      " |-- kid-friendly: double (nullable = true)\n",
      " |-- kidney friendly: double (nullable = true)\n",
      " |-- kirsch: double (nullable = true)\n",
      " |-- kitchen olympics: double (nullable = true)\n",
      " |-- kiwi: double (nullable = true)\n",
      " |-- kosher: double (nullable = true)\n",
      " |-- kosher for passover: double (nullable = true)\n",
      " |-- kumquat: double (nullable = true)\n",
      " |-- kwanzaa: double (nullable = true)\n",
      " |-- labor day: double (nullable = true)\n",
      " |-- lamb: double (nullable = true)\n",
      " |-- lamb chop: double (nullable = true)\n",
      " |-- lamb shank: double (nullable = true)\n",
      " |-- lancaster: double (nullable = true)\n",
      " |-- las vegas: double (nullable = true)\n",
      " |-- lasagna: double (nullable = true)\n",
      " |-- leafy green: double (nullable = true)\n",
      " |-- leek: double (nullable = true)\n",
      " |-- legume: double (nullable = true)\n",
      " |-- lemon: double (nullable = true)\n",
      " |-- lemon juice: double (nullable = true)\n",
      " |-- lemongrass: double (nullable = true)\n",
      " |-- lentil: double (nullable = true)\n",
      " |-- lettuce: double (nullable = true)\n",
      " |-- lima bean: double (nullable = true)\n",
      " |-- lime: double (nullable = true)\n",
      " |-- lime juice: double (nullable = true)\n",
      " |-- lingonberry: double (nullable = true)\n",
      " |-- liqueur: double (nullable = true)\n",
      " |-- lobster: double (nullable = true)\n",
      " |-- london: double (nullable = true)\n",
      " |-- long beach: double (nullable = true)\n",
      " |-- los angeles: double (nullable = true)\n",
      " |-- louisiana: double (nullable = true)\n",
      " |-- louisville: double (nullable = true)\n",
      " |-- low cal: double (nullable = true)\n",
      " |-- low carb: double (nullable = true)\n",
      " |-- low cholesterol: double (nullable = true)\n",
      " |-- low fat: double (nullable = true)\n",
      " |-- low sodium: double (nullable = true)\n",
      " |-- low sugar: double (nullable = true)\n",
      " |-- low/no sugar: double (nullable = true)\n",
      " |-- lunar new year: double (nullable = true)\n",
      " |-- lunch: double (nullable = true)\n",
      " |-- lychee: double (nullable = true)\n",
      " |-- macadamia nut: double (nullable = true)\n",
      " |-- macaroni and cheese: double (nullable = true)\n",
      " |-- maine: double (nullable = true)\n",
      " |-- mandoline: double (nullable = true)\n",
      " |-- mango: double (nullable = true)\n",
      " |-- maple syrup: double (nullable = true)\n",
      " |-- mardi gras: double (nullable = true)\n",
      " |-- margarita: double (nullable = true)\n",
      " |-- marinade: double (nullable = true)\n",
      " |-- marinate: double (nullable = true)\n",
      " |-- marsala: double (nullable = true)\n",
      " |-- marscarpone: double (nullable = true)\n",
      " |-- marshmallow: double (nullable = true)\n",
      " |-- martini: double (nullable = true)\n",
      " |-- maryland: double (nullable = true)\n",
      " |-- massachusetts: double (nullable = true)\n",
      " |-- mayonnaise: double (nullable = true)\n",
      " |-- meat: double (nullable = true)\n",
      " |-- meatball: double (nullable = true)\n",
      " |-- meatloaf: double (nullable = true)\n",
      " |-- melon: double (nullable = true)\n",
      " |-- mexico: double (nullable = true)\n",
      " |-- mezcal: double (nullable = true)\n",
      " |-- miami: double (nullable = true)\n",
      " |-- michigan: double (nullable = true)\n",
      " |-- microwave: double (nullable = true)\n",
      " |-- midori: double (nullable = true)\n",
      " |-- milk/cream: double (nullable = true)\n",
      " |-- minneapolis: double (nullable = true)\n",
      " |-- minnesota: double (nullable = true)\n",
      " |-- mint: double (nullable = true)\n",
      " |-- mississippi: double (nullable = true)\n",
      " |-- missouri: double (nullable = true)\n",
      " |-- mixer: double (nullable = true)\n",
      " |-- molasses: double (nullable = true)\n",
      " |-- monterey jack: double (nullable = true)\n",
      " |-- mortar and pestle: double (nullable = true)\n",
      " |-- mother's day: double (nullable = true)\n",
      " |-- mozzarella: double (nullable = true)\n",
      " |-- muffin: double (nullable = true)\n",
      " |-- mushroom: double (nullable = true)\n",
      " |-- mussel: double (nullable = true)\n",
      " |-- mustard: double (nullable = true)\n",
      " |-- mustard greens: double (nullable = true)\n",
      " |-- nancy silverton: double (nullable = true)\n",
      " |-- nebraska: double (nullable = true)\n",
      " |-- nectarine: double (nullable = true)\n",
      " |-- new hampshire: double (nullable = true)\n",
      " |-- new jersey: double (nullable = true)\n",
      " |-- new mexico: double (nullable = true)\n",
      " |-- new orleans: double (nullable = true)\n",
      " |-- new year's day: double (nullable = true)\n",
      " |-- new year's eve: double (nullable = true)\n",
      " |-- new york: double (nullable = true)\n",
      " |-- no meat, no problem: double (nullable = true)\n",
      " |-- no sugar added: double (nullable = true)\n",
      " |-- no-cook: double (nullable = true)\n",
      " |-- non-alcoholic: double (nullable = true)\n",
      " |-- noodle: double (nullable = true)\n",
      " |-- north carolina: double (nullable = true)\n",
      " |-- nut: double (nullable = true)\n",
      " |-- nutmeg: double (nullable = true)\n",
      " |-- oat: double (nullable = true)\n",
      " |-- oatmeal: double (nullable = true)\n",
      " |-- octopus: double (nullable = true)\n",
      " |-- ohio: double (nullable = true)\n",
      " |-- oklahoma: double (nullable = true)\n",
      " |-- okra: double (nullable = true)\n",
      " |-- oktoberfest: double (nullable = true)\n",
      " |-- olive: double (nullable = true)\n",
      " |-- omelet: double (nullable = true)\n",
      " |-- one-pot meal: double (nullable = true)\n",
      " |-- onion: double (nullable = true)\n",
      " |-- orange: double (nullable = true)\n",
      " |-- orange juice: double (nullable = true)\n",
      " |-- oregano: double (nullable = true)\n",
      " |-- oregon: double (nullable = true)\n",
      " |-- organic: double (nullable = true)\n",
      " |-- orzo: double (nullable = true)\n",
      " |-- oscars: double (nullable = true)\n",
      " |-- oyster: double (nullable = true)\n",
      " |-- pacific palisades: double (nullable = true)\n",
      " |-- paleo: double (nullable = true)\n",
      " |-- pan-fry: double (nullable = true)\n",
      " |-- pancake: double (nullable = true)\n",
      " |-- papaya: double (nullable = true)\n",
      " |-- paprika: double (nullable = true)\n",
      " |-- parade: double (nullable = true)\n",
      " |-- paris: double (nullable = true)\n",
      " |-- parmesan: double (nullable = true)\n",
      " |-- parsley: double (nullable = true)\n",
      " |-- parsnip: double (nullable = true)\n",
      " |-- party: double (nullable = true)\n",
      " |-- pasadena: double (nullable = true)\n",
      " |-- passion fruit: double (nullable = true)\n",
      " |-- passover: double (nullable = true)\n",
      " |-- pasta: double (nullable = true)\n",
      " |-- pasta maker: double (nullable = true)\n",
      " |-- pastry: double (nullable = true)\n",
      " |-- pea: double (nullable = true)\n",
      " |-- peach: double (nullable = true)\n",
      " |-- peanut: double (nullable = true)\n",
      " |-- peanut butter: double (nullable = true)\n",
      " |-- peanut free: double (nullable = true)\n",
      " |-- pear: double (nullable = true)\n",
      " |-- pecan: double (nullable = true)\n",
      " |-- pennsylvania: double (nullable = true)\n",
      " |-- pepper: double (nullable = true)\n",
      " |-- pernod: double (nullable = true)\n",
      " |-- persian new year: double (nullable = true)\n",
      " |-- persimmon: double (nullable = true)\n",
      " |-- peru: double (nullable = true)\n",
      " |-- pescatarian: double (nullable = true)\n",
      " |-- philippines: double (nullable = true)\n",
      " |-- phyllo/puff pastry dough: double (nullable = true)\n",
      " |-- pickles: double (nullable = true)\n",
      " |-- picnic: double (nullable = true)\n",
      " |-- pie: double (nullable = true)\n",
      " |-- pine nut: double (nullable = true)\n",
      " |-- pineapple: double (nullable = true)\n",
      " |-- pistachio: double (nullable = true)\n",
      " |-- pittsburgh: double (nullable = true)\n",
      " |-- pizza: double (nullable = true)\n",
      " |-- plantain: double (nullable = true)\n",
      " |-- plum: double (nullable = true)\n",
      " |-- poach: double (nullable = true)\n",
      " |-- poblano: double (nullable = true)\n",
      " |-- poker/game night: double (nullable = true)\n",
      " |-- pomegranate: double (nullable = true)\n",
      " |-- pomegranate juice: double (nullable = true)\n",
      " |-- poppy: double (nullable = true)\n",
      " |-- pork: double (nullable = true)\n",
      " |-- pork chop: double (nullable = true)\n",
      " |-- pork rib: double (nullable = true)\n",
      " |-- pork tenderloin: double (nullable = true)\n",
      " |-- port: double (nullable = true)\n",
      " |-- portland: double (nullable = true)\n",
      " |-- pot pie: double (nullable = true)\n",
      " |-- potato: double (nullable = true)\n",
      " |-- potato salad: double (nullable = true)\n",
      " |-- potluck: double (nullable = true)\n",
      " |-- poultry: double (nullable = true)\n",
      " |-- poultry sausage: double (nullable = true)\n",
      " |-- pressure cooker: double (nullable = true)\n",
      " |-- prosciutto: double (nullable = true)\n",
      " |-- providence: double (nullable = true)\n",
      " |-- prune: double (nullable = true)\n",
      " |-- pumpkin: double (nullable = true)\n",
      " |-- punch: double (nullable = true)\n",
      " |-- purim: double (nullable = true)\n",
      " |-- quail: double (nullable = true)\n",
      " |-- quiche: double (nullable = true)\n",
      " |-- quick & easy: double (nullable = true)\n",
      " |-- quick and healthy: double (nullable = true)\n",
      " |-- quince: double (nullable = true)\n",
      " |-- quinoa: double (nullable = true)\n",
      " |-- rabbit: double (nullable = true)\n",
      " |-- rack of lamb: double (nullable = true)\n",
      " |-- radicchio: double (nullable = true)\n",
      " |-- radish: double (nullable = true)\n",
      " |-- raisin: double (nullable = true)\n",
      " |-- ramadan: double (nullable = true)\n",
      " |-- ramekin: double (nullable = true)\n",
      " |-- raspberry: double (nullable = true)\n",
      " |-- raw: double (nullable = true)\n",
      " |-- red wine: double (nullable = true)\n",
      " |-- rhode island: double (nullable = true)\n",
      " |-- rhubarb: double (nullable = true)\n",
      " |-- rice: double (nullable = true)\n",
      " |-- ricotta: double (nullable = true)\n",
      " |-- roast: double (nullable = true)\n",
      " |-- root vegetable: double (nullable = true)\n",
      " |-- rosemary: double (nullable = true)\n",
      " |-- rosh hashanah/yom kippur: double (nullable = true)\n",
      " |-- rosé: double (nullable = true)\n",
      " |-- rub: double (nullable = true)\n",
      " |-- rum: double (nullable = true)\n",
      " |-- rutabaga: double (nullable = true)\n",
      " |-- rye: double (nullable = true)\n",
      " |-- saffron: double (nullable = true)\n",
      " |-- sage: double (nullable = true)\n",
      " |-- sake: double (nullable = true)\n",
      " |-- salad: double (nullable = true)\n",
      " |-- salad dressing: double (nullable = true)\n",
      " |-- salmon: double (nullable = true)\n",
      " |-- salsa: double (nullable = true)\n",
      " |-- san francisco: double (nullable = true)\n",
      " |-- sandwich: double (nullable = true)\n",
      " |-- sandwich theory: double (nullable = true)\n",
      " |-- sangria: double (nullable = true)\n",
      " |-- santa monica: double (nullable = true)\n",
      " |-- sardine: double (nullable = true)\n",
      " |-- sauce: double (nullable = true)\n",
      " |-- sausage: double (nullable = true)\n",
      " |-- sauté: double (nullable = true)\n",
      " |-- scallop: double (nullable = true)\n",
      " |-- scotch: double (nullable = true)\n",
      " |-- seafood: double (nullable = true)\n",
      " |-- seattle: double (nullable = true)\n",
      " |-- seed: double (nullable = true)\n",
      " |-- self: double (nullable = true)\n",
      " |-- semolina: double (nullable = true)\n",
      " |-- sesame: double (nullable = true)\n",
      " |-- sesame oil: double (nullable = true)\n",
      " |-- shallot: double (nullable = true)\n",
      " |-- shavuot: double (nullable = true)\n",
      " |-- shellfish: double (nullable = true)\n",
      " |-- sherry: double (nullable = true)\n",
      " |-- shower: double (nullable = true)\n",
      " |-- shrimp: double (nullable = true)\n",
      " |-- side: double (nullable = true)\n",
      " |-- simmer: double (nullable = true)\n",
      " |-- skewer: double (nullable = true)\n",
      " |-- slow cooker: double (nullable = true)\n",
      " |-- smoker: double (nullable = true)\n",
      " |-- smoothie: double (nullable = true)\n",
      " |-- snapper: double (nullable = true)\n",
      " |-- sorbet: double (nullable = true)\n",
      " |-- soufflé/meringue: double (nullable = true)\n",
      " |-- soup/stew: double (nullable = true)\n",
      " |-- sour cream: double (nullable = true)\n",
      " |-- sourdough: double (nullable = true)\n",
      " |-- south carolina: double (nullable = true)\n",
      " |-- soy: double (nullable = true)\n",
      " |-- soy free: double (nullable = true)\n",
      " |-- soy sauce: double (nullable = true)\n",
      " |-- spain: double (nullable = true)\n",
      " |-- sparkling wine: double (nullable = true)\n",
      " |-- spice: double (nullable = true)\n",
      " |-- spinach: double (nullable = true)\n",
      " |-- spirit: double (nullable = true)\n",
      " |-- spring: double (nullable = true)\n",
      " |-- spritzer: double (nullable = true)\n",
      " |-- squash: double (nullable = true)\n",
      " |-- squid: double (nullable = true)\n",
      " |-- st. louis: double (nullable = true)\n",
      " |-- st. patrick's day: double (nullable = true)\n",
      " |-- steak: double (nullable = true)\n",
      " |-- steam: double (nullable = true)\n",
      " |-- stew: double (nullable = true)\n",
      " |-- stir-fry: double (nullable = true)\n",
      " |-- stock: double (nullable = true)\n",
      " |-- strawberry: double (nullable = true)\n",
      " |-- stuffing/dressing: double (nullable = true)\n",
      " |-- sugar conscious: double (nullable = true)\n",
      " |-- sugar snap pea: double (nullable = true)\n",
      " |-- sukkot: double (nullable = true)\n",
      " |-- summer: double (nullable = true)\n",
      " |-- super bowl: double (nullable = true)\n",
      " |-- suzanne goin: double (nullable = true)\n",
      " |-- sweet potato/yam: double (nullable = true)\n",
      " |-- swiss cheese: double (nullable = true)\n",
      " |-- switzerland: double (nullable = true)\n",
      " |-- swordfish: double (nullable = true)\n",
      " |-- taco: double (nullable = true)\n",
      " |-- tailgating: double (nullable = true)\n",
      " |-- tamarind: double (nullable = true)\n",
      " |-- tangerine: double (nullable = true)\n",
      " |-- tapioca: double (nullable = true)\n",
      " |-- tarragon: double (nullable = true)\n",
      " |-- tart: double (nullable = true)\n",
      " |-- tea: double (nullable = true)\n",
      " |-- tennessee: double (nullable = true)\n",
      " |-- tequila: double (nullable = true)\n",
      " |-- tested & improved: double (nullable = true)\n",
      " |-- texas: double (nullable = true)\n",
      " |-- thanksgiving: double (nullable = true)\n",
      " |-- thyme: double (nullable = true)\n",
      " |-- tilapia: double (nullable = true)\n",
      " |-- tofu: double (nullable = true)\n",
      " |-- tomatillo: double (nullable = true)\n",
      " |-- tomato: double (nullable = true)\n",
      " |-- tortillas: double (nullable = true)\n",
      " |-- tree nut: double (nullable = true)\n",
      " |-- tree nut free: double (nullable = true)\n",
      " |-- triple sec: double (nullable = true)\n",
      " |-- tropical fruit: double (nullable = true)\n",
      " |-- trout: double (nullable = true)\n",
      " |-- tuna: double (nullable = true)\n",
      " |-- turnip: double (nullable = true)\n",
      " |-- utah: double (nullable = true)\n",
      " |-- valentine's day: double (nullable = true)\n",
      " |-- vanilla: double (nullable = true)\n",
      " |-- veal: double (nullable = true)\n",
      " |-- vegan: double (nullable = true)\n",
      " |-- vegetable: double (nullable = true)\n",
      " |-- vegetarian: double (nullable = true)\n",
      " |-- venison: double (nullable = true)\n",
      " |-- vermont: double (nullable = true)\n",
      " |-- vermouth: double (nullable = true)\n",
      " |-- vinegar: double (nullable = true)\n",
      " |-- virginia: double (nullable = true)\n",
      " |-- vodka: double (nullable = true)\n",
      " |-- waffle: double (nullable = true)\n",
      " |-- walnut: double (nullable = true)\n",
      " |-- wasabi: double (nullable = true)\n",
      " |-- washington: double (nullable = true)\n",
      " |-- washington, d.c.: double (nullable = true)\n",
      " |-- watercress: double (nullable = true)\n",
      " |-- watermelon: double (nullable = true)\n",
      " |-- wedding: double (nullable = true)\n",
      " |-- weelicious: double (nullable = true)\n",
      " |-- west virginia: double (nullable = true)\n",
      " |-- westwood: double (nullable = true)\n",
      " |-- wheat/gluten-free: double (nullable = true)\n",
      " |-- whiskey: double (nullable = true)\n",
      " |-- white wine: double (nullable = true)\n",
      " |-- whole wheat: double (nullable = true)\n",
      " |-- wild rice: double (nullable = true)\n",
      " |-- windsor: double (nullable = true)\n",
      " |-- wine: double (nullable = true)\n",
      " |-- winter: double (nullable = true)\n",
      " |-- wisconsin: double (nullable = true)\n",
      " |-- wok: double (nullable = true)\n",
      " |-- yellow squash: double (nullable = true)\n",
      " |-- yogurt: double (nullable = true)\n",
      " |-- yonkers: double (nullable = true)\n",
      " |-- yuca: double (nullable = true)\n",
      " |-- zucchini: double (nullable = true)\n",
      " |-- cookbooks: double (nullable = true)\n",
      " |-- leftovers: double (nullable = true)\n",
      " |-- snack: double (nullable = true)\n",
      " |-- snack week: double (nullable = true)\n",
      " |-- turkey: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "file_path = r\"D:\\bigdata\\spark_ practice\\DataAnalysisWithPythonAndPySpark-Data-trunk\\recipes\\epi_r.csv\"\n",
    "food = spark.read.csv(file_path,inferSchema=True,header=True)\n",
    "print(food.count(),len(food.columns))\n",
    "food.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "015bb703-e846-48d5-8325-9aa22ffc06b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.sql.dataframe.DataFrame"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(food)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "15070003-45de-476a-a3b9-e4c6216ea4c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanitize_column_name(name):\n",
    "    answer = name\n",
    "    for i,j in ((\" \",\"_\"),(\"-\",\"_\"),(\"&\",\"and\"),(\"/\",\"_\")):\n",
    "        answer = answer.replace(i,j)\n",
    "    return \"\".join([char for char in answer if char.isalpha() or char.isdigit() or char==\"_\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b47c44c3-86c1-4a0b-8752-13351985acd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method toDF in module pyspark.sql.dataframe:\n",
      "\n",
      "toDF(*cols: str) -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Returns a new :class:`DataFrame` that with new specified column names\n",
      "    \n",
      "    .. versionadded:: 1.6.0\n",
      "    \n",
      "    .. versionchanged:: 3.4.0\n",
      "        Supports Spark Connect.\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    *cols : tuple\n",
      "        a tuple of string new column name. The length of the\n",
      "        list needs to be the same as the number of columns in the initial\n",
      "        :class:`DataFrame`\n",
      "    \n",
      "    Returns\n",
      "    -------\n",
      "    :class:`DataFrame`\n",
      "        DataFrame with new column names.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df = spark.createDataFrame([(14, \"Tom\"), (23, \"Alice\"),\n",
      "    ...     (16, \"Bob\")], [\"age\", \"name\"])\n",
      "    >>> df.toDF('f1', 'f2').show()\n",
      "    +---+-----+\n",
      "    | f1|   f2|\n",
      "    +---+-----+\n",
      "    | 14|  Tom|\n",
      "    | 23|Alice|\n",
      "    | 16|  Bob|\n",
      "    +---+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(food.toDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "50d6407e-4ed2-4666-9fd9-547eb653b00b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- title: string (nullable = true)\n",
      " |-- rating: string (nullable = true)\n",
      " |-- calories: string (nullable = true)\n",
      " |-- protein: double (nullable = true)\n",
      " |-- fat: double (nullable = true)\n",
      " |-- sodium: double (nullable = true)\n",
      " |-- cakeweek: double (nullable = true)\n",
      " |-- wasteless: double (nullable = true)\n",
      " |-- 22_minute_meals: double (nullable = true)\n",
      " |-- 3_ingredient_recipes: double (nullable = true)\n",
      " |-- 30_days_of_groceries: double (nullable = true)\n",
      " |-- advance_prep_required: double (nullable = true)\n",
      " |-- alabama: double (nullable = true)\n",
      " |-- alaska: double (nullable = true)\n",
      " |-- alcoholic: double (nullable = true)\n",
      " |-- almond: double (nullable = true)\n",
      " |-- amaretto: double (nullable = true)\n",
      " |-- anchovy: double (nullable = true)\n",
      " |-- anise: double (nullable = true)\n",
      " |-- anniversary: double (nullable = true)\n",
      " |-- anthony_bourdain: double (nullable = true)\n",
      " |-- aperitif: double (nullable = true)\n",
      " |-- appetizer: double (nullable = true)\n",
      " |-- apple: double (nullable = true)\n",
      " |-- apple_juice: double (nullable = true)\n",
      " |-- apricot: double (nullable = true)\n",
      " |-- arizona: double (nullable = true)\n",
      " |-- artichoke: double (nullable = true)\n",
      " |-- arugula: double (nullable = true)\n",
      " |-- asian_pear: double (nullable = true)\n",
      " |-- asparagus: double (nullable = true)\n",
      " |-- aspen: double (nullable = true)\n",
      " |-- atlanta: double (nullable = true)\n",
      " |-- australia: double (nullable = true)\n",
      " |-- avocado: double (nullable = true)\n",
      " |-- back_to_school: double (nullable = true)\n",
      " |-- backyard_bbq: double (nullable = true)\n",
      " |-- bacon: double (nullable = true)\n",
      " |-- bake: double (nullable = true)\n",
      " |-- banana: double (nullable = true)\n",
      " |-- barley: double (nullable = true)\n",
      " |-- basil: double (nullable = true)\n",
      " |-- bass: double (nullable = true)\n",
      " |-- bastille_day: double (nullable = true)\n",
      " |-- bean: double (nullable = true)\n",
      " |-- beef: double (nullable = true)\n",
      " |-- beef_rib: double (nullable = true)\n",
      " |-- beef_shank: double (nullable = true)\n",
      " |-- beef_tenderloin: double (nullable = true)\n",
      " |-- beer: double (nullable = true)\n",
      " |-- beet: double (nullable = true)\n",
      " |-- bell_pepper: double (nullable = true)\n",
      " |-- berry: double (nullable = true)\n",
      " |-- beverly_hills: double (nullable = true)\n",
      " |-- birthday: double (nullable = true)\n",
      " |-- biscuit: double (nullable = true)\n",
      " |-- bitters: double (nullable = true)\n",
      " |-- blackberry: double (nullable = true)\n",
      " |-- blender: double (nullable = true)\n",
      " |-- blue_cheese: double (nullable = true)\n",
      " |-- blueberry: double (nullable = true)\n",
      " |-- boil: double (nullable = true)\n",
      " |-- bok_choy: double (nullable = true)\n",
      " |-- bon_appétit: double (nullable = true)\n",
      " |-- bon_apptit: double (nullable = true)\n",
      " |-- boston: double (nullable = true)\n",
      " |-- bourbon: double (nullable = true)\n",
      " |-- braise: double (nullable = true)\n",
      " |-- bran: double (nullable = true)\n",
      " |-- brandy: double (nullable = true)\n",
      " |-- bread: double (nullable = true)\n",
      " |-- breadcrumbs: double (nullable = true)\n",
      " |-- breakfast: double (nullable = true)\n",
      " |-- brie: double (nullable = true)\n",
      " |-- brine: double (nullable = true)\n",
      " |-- brisket: double (nullable = true)\n",
      " |-- broccoli: double (nullable = true)\n",
      " |-- broccoli_rabe: double (nullable = true)\n",
      " |-- broil: double (nullable = true)\n",
      " |-- brooklyn: double (nullable = true)\n",
      " |-- brown_rice: double (nullable = true)\n",
      " |-- brownie: double (nullable = true)\n",
      " |-- brunch: double (nullable = true)\n",
      " |-- brussel_sprout: double (nullable = true)\n",
      " |-- buffalo: double (nullable = true)\n",
      " |-- buffet: double (nullable = true)\n",
      " |-- bulgaria: double (nullable = true)\n",
      " |-- bulgur: double (nullable = true)\n",
      " |-- burrito: double (nullable = true)\n",
      " |-- butter: double (nullable = true)\n",
      " |-- buttermilk: double (nullable = true)\n",
      " |-- butternut_squash: double (nullable = true)\n",
      " |-- butterscotch_caramel: double (nullable = true)\n",
      " |-- cabbage: double (nullable = true)\n",
      " |-- cake: double (nullable = true)\n",
      " |-- california: double (nullable = true)\n",
      " |-- calvados: double (nullable = true)\n",
      " |-- cambridge: double (nullable = true)\n",
      " |-- campari: double (nullable = true)\n",
      " |-- camping: double (nullable = true)\n",
      " |-- canada: double (nullable = true)\n",
      " |-- candy: double (nullable = true)\n",
      " |-- candy_thermometer: double (nullable = true)\n",
      " |-- cantaloupe: double (nullable = true)\n",
      " |-- capers: double (nullable = true)\n",
      " |-- caraway: double (nullable = true)\n",
      " |-- cardamom: double (nullable = true)\n",
      " |-- carrot: double (nullable = true)\n",
      " |-- cashew: double (nullable = true)\n",
      " |-- casserole_gratin: double (nullable = true)\n",
      " |-- cauliflower: double (nullable = true)\n",
      " |-- caviar: double (nullable = true)\n",
      " |-- celery: double (nullable = true)\n",
      " |-- chambord: double (nullable = true)\n",
      " |-- champagne: double (nullable = true)\n",
      " |-- chard: double (nullable = true)\n",
      " |-- chartreuse: double (nullable = true)\n",
      " |-- cheddar: double (nullable = true)\n",
      " |-- cheese: double (nullable = true)\n",
      " |-- cherry: double (nullable = true)\n",
      " |-- chestnut: double (nullable = true)\n",
      " |-- chicago: double (nullable = true)\n",
      " |-- chicken: double (nullable = true)\n",
      " |-- chickpea: double (nullable = true)\n",
      " |-- chile: double (nullable = true)\n",
      " |-- chile_pepper: double (nullable = true)\n",
      " |-- chili: double (nullable = true)\n",
      " |-- chill: double (nullable = true)\n",
      " |-- chive: double (nullable = true)\n",
      " |-- chocolate: double (nullable = true)\n",
      " |-- christmas: double (nullable = true)\n",
      " |-- christmas_eve: double (nullable = true)\n",
      " |-- cilantro: double (nullable = true)\n",
      " |-- cinco_de_mayo: double (nullable = true)\n",
      " |-- cinnamon: double (nullable = true)\n",
      " |-- citrus: double (nullable = true)\n",
      " |-- clam: double (nullable = true)\n",
      " |-- clove: double (nullable = true)\n",
      " |-- cobbler_crumble: double (nullable = true)\n",
      " |-- cocktail: double (nullable = true)\n",
      " |-- cocktail_party: double (nullable = true)\n",
      " |-- coconut: double (nullable = true)\n",
      " |-- cod: double (nullable = true)\n",
      " |-- coffee: double (nullable = true)\n",
      " |-- coffee_grinder: double (nullable = true)\n",
      " |-- cognac_armagnac: double (nullable = true)\n",
      " |-- collard_greens: double (nullable = true)\n",
      " |-- colorado: double (nullable = true)\n",
      " |-- columbus: double (nullable = true)\n",
      " |-- condiment: double (nullable = true)\n",
      " |-- condiment_spread: double (nullable = true)\n",
      " |-- connecticut: double (nullable = true)\n",
      " |-- cook_like_a_diner: double (nullable = true)\n",
      " |-- cookbook_critic: double (nullable = true)\n",
      " |-- cookie: double (nullable = true)\n",
      " |-- cookies: double (nullable = true)\n",
      " |-- coriander: double (nullable = true)\n",
      " |-- corn: double (nullable = true)\n",
      " |-- cornmeal: double (nullable = true)\n",
      " |-- costa_mesa: double (nullable = true)\n",
      " |-- cottage_cheese: double (nullable = true)\n",
      " |-- couscous: double (nullable = true)\n",
      " |-- crab: double (nullable = true)\n",
      " |-- cranberry: double (nullable = true)\n",
      " |-- cranberry_sauce: double (nullable = true)\n",
      " |-- cream_cheese: double (nullable = true)\n",
      " |-- créme_de_cacao: double (nullable = true)\n",
      " |-- crêpe: double (nullable = true)\n",
      " |-- crme_de_cacao: double (nullable = true)\n",
      " |-- cuba: double (nullable = true)\n",
      " |-- cucumber: double (nullable = true)\n",
      " |-- cumin: double (nullable = true)\n",
      " |-- cupcake: double (nullable = true)\n",
      " |-- currant: double (nullable = true)\n",
      " |-- curry: double (nullable = true)\n",
      " |-- custard: double (nullable = true)\n",
      " |-- dairy: double (nullable = true)\n",
      " |-- dairy_free: double (nullable = true)\n",
      " |-- dallas: double (nullable = true)\n",
      " |-- date: double (nullable = true)\n",
      " |-- deep_fry: double (nullable = true)\n",
      " |-- denver: double (nullable = true)\n",
      " |-- dessert: double (nullable = true)\n",
      " |-- digestif: double (nullable = true)\n",
      " |-- dill: double (nullable = true)\n",
      " |-- dinner: double (nullable = true)\n",
      " |-- dip: double (nullable = true)\n",
      " |-- diwali: double (nullable = true)\n",
      " |-- dominican_republic: double (nullable = true)\n",
      " |-- dorie_greenspan: double (nullable = true)\n",
      " |-- double_boiler: double (nullable = true)\n",
      " |-- dried_fruit: double (nullable = true)\n",
      " |-- drink: double (nullable = true)\n",
      " |-- drinks: double (nullable = true)\n",
      " |-- duck: double (nullable = true)\n",
      " |-- easter: double (nullable = true)\n",
      " |-- eau_de_vie: double (nullable = true)\n",
      " |-- edible_gift: double (nullable = true)\n",
      " |-- egg: double (nullable = true)\n",
      " |-- egg_nog: double (nullable = true)\n",
      " |-- eggplant: double (nullable = true)\n",
      " |-- egypt: double (nullable = true)\n",
      " |-- emeril_lagasse: double (nullable = true)\n",
      " |-- endive: double (nullable = true)\n",
      " |-- engagement_party: double (nullable = true)\n",
      " |-- england: double (nullable = true)\n",
      " |-- entertaining: double (nullable = true)\n",
      " |-- epi__ushg: double (nullable = true)\n",
      " |-- epi_loves_the_microwave: double (nullable = true)\n",
      " |-- escarole: double (nullable = true)\n",
      " |-- fall: double (nullable = true)\n",
      " |-- family_reunion: double (nullable = true)\n",
      " |-- fat_free: double (nullable = true)\n",
      " |-- fathers_day: double (nullable = true)\n",
      " |-- fennel: double (nullable = true)\n",
      " |-- feta: double (nullable = true)\n",
      " |-- fig: double (nullable = true)\n",
      " |-- fish: double (nullable = true)\n",
      " |-- flaming_hot_summer: double (nullable = true)\n",
      " |-- flat_bread: double (nullable = true)\n",
      " |-- florida: double (nullable = true)\n",
      " |-- fontina: double (nullable = true)\n",
      " |-- food_processor: double (nullable = true)\n",
      " |-- fortified_wine: double (nullable = true)\n",
      " |-- fourth_of_july: double (nullable = true)\n",
      " |-- france: double (nullable = true)\n",
      " |-- frangelico: double (nullable = true)\n",
      " |-- frankenrecipe: double (nullable = true)\n",
      " |-- freeze_chill: double (nullable = true)\n",
      " |-- freezer_food: double (nullable = true)\n",
      " |-- friendsgiving: double (nullable = true)\n",
      " |-- frittata: double (nullable = true)\n",
      " |-- fritter: double (nullable = true)\n",
      " |-- frozen_dessert: double (nullable = true)\n",
      " |-- fruit: double (nullable = true)\n",
      " |-- fruit_juice: double (nullable = true)\n",
      " |-- fry: double (nullable = true)\n",
      " |-- game: double (nullable = true)\n",
      " |-- garlic: double (nullable = true)\n",
      " |-- georgia: double (nullable = true)\n",
      " |-- germany: double (nullable = true)\n",
      " |-- gin: double (nullable = true)\n",
      " |-- ginger: double (nullable = true)\n",
      " |-- goat_cheese: double (nullable = true)\n",
      " |-- goose: double (nullable = true)\n",
      " |-- gouda: double (nullable = true)\n",
      " |-- gourmet: double (nullable = true)\n",
      " |-- graduation: double (nullable = true)\n",
      " |-- grains: double (nullable = true)\n",
      " |-- grand_marnier: double (nullable = true)\n",
      " |-- granola: double (nullable = true)\n",
      " |-- grape: double (nullable = true)\n",
      " |-- grapefruit: double (nullable = true)\n",
      " |-- grappa: double (nullable = true)\n",
      " |-- green_bean: double (nullable = true)\n",
      " |-- green_onion_scallion: double (nullable = true)\n",
      " |-- grill: double (nullable = true)\n",
      " |-- grill_barbecue: double (nullable = true)\n",
      " |-- ground_beef: double (nullable = true)\n",
      " |-- ground_lamb: double (nullable = true)\n",
      " |-- guam: double (nullable = true)\n",
      " |-- guava: double (nullable = true)\n",
      " |-- haiti: double (nullable = true)\n",
      " |-- halibut: double (nullable = true)\n",
      " |-- halloween: double (nullable = true)\n",
      " |-- ham: double (nullable = true)\n",
      " |-- hamburger: double (nullable = true)\n",
      " |-- hanukkah: double (nullable = true)\n",
      " |-- harpercollins: double (nullable = true)\n",
      " |-- hawaii: double (nullable = true)\n",
      " |-- hazelnut: double (nullable = true)\n",
      " |-- healdsburg: double (nullable = true)\n",
      " |-- healthy: double (nullable = true)\n",
      " |-- herb: double (nullable = true)\n",
      " |-- high_fiber: double (nullable = true)\n",
      " |-- hollywood: double (nullable = true)\n",
      " |-- hominy_cornmeal_masa: double (nullable = true)\n",
      " |-- honey: double (nullable = true)\n",
      " |-- honeydew: double (nullable = true)\n",
      " |-- hors_doeuvre: double (nullable = true)\n",
      " |-- horseradish: double (nullable = true)\n",
      " |-- hot_drink: double (nullable = true)\n",
      " |-- hot_pepper: double (nullable = true)\n",
      " |-- house_and_garden: double (nullable = true)\n",
      " |-- house_cocktail: double (nullable = true)\n",
      " |-- houston: double (nullable = true)\n",
      " |-- hummus: double (nullable = true)\n",
      " |-- ice_cream: double (nullable = true)\n",
      " |-- ice_cream_machine: double (nullable = true)\n",
      " |-- iced_coffee: double (nullable = true)\n",
      " |-- iced_tea: double (nullable = true)\n",
      " |-- idaho: double (nullable = true)\n",
      " |-- illinois: double (nullable = true)\n",
      " |-- indiana: double (nullable = true)\n",
      " |-- iowa: double (nullable = true)\n",
      " |-- ireland: double (nullable = true)\n",
      " |-- israel: double (nullable = true)\n",
      " |-- italy: double (nullable = true)\n",
      " |-- jalapeño: double (nullable = true)\n",
      " |-- jam_or_jelly: double (nullable = true)\n",
      " |-- jamaica: double (nullable = true)\n",
      " |-- japan: double (nullable = true)\n",
      " |-- jerusalem_artichoke: double (nullable = true)\n",
      " |-- juicer: double (nullable = true)\n",
      " |-- jícama: double (nullable = true)\n",
      " |-- kahlúa: double (nullable = true)\n",
      " |-- kale: double (nullable = true)\n",
      " |-- kansas: double (nullable = true)\n",
      " |-- kansas_city: double (nullable = true)\n",
      " |-- kentucky: double (nullable = true)\n",
      " |-- kentucky_derby: double (nullable = true)\n",
      " |-- kid_friendly: double (nullable = true)\n",
      " |-- kidney_friendly: double (nullable = true)\n",
      " |-- kirsch: double (nullable = true)\n",
      " |-- kitchen_olympics: double (nullable = true)\n",
      " |-- kiwi: double (nullable = true)\n",
      " |-- kosher: double (nullable = true)\n",
      " |-- kosher_for_passover: double (nullable = true)\n",
      " |-- kumquat: double (nullable = true)\n",
      " |-- kwanzaa: double (nullable = true)\n",
      " |-- labor_day: double (nullable = true)\n",
      " |-- lamb: double (nullable = true)\n",
      " |-- lamb_chop: double (nullable = true)\n",
      " |-- lamb_shank: double (nullable = true)\n",
      " |-- lancaster: double (nullable = true)\n",
      " |-- las_vegas: double (nullable = true)\n",
      " |-- lasagna: double (nullable = true)\n",
      " |-- leafy_green: double (nullable = true)\n",
      " |-- leek: double (nullable = true)\n",
      " |-- legume: double (nullable = true)\n",
      " |-- lemon: double (nullable = true)\n",
      " |-- lemon_juice: double (nullable = true)\n",
      " |-- lemongrass: double (nullable = true)\n",
      " |-- lentil: double (nullable = true)\n",
      " |-- lettuce: double (nullable = true)\n",
      " |-- lima_bean: double (nullable = true)\n",
      " |-- lime: double (nullable = true)\n",
      " |-- lime_juice: double (nullable = true)\n",
      " |-- lingonberry: double (nullable = true)\n",
      " |-- liqueur: double (nullable = true)\n",
      " |-- lobster: double (nullable = true)\n",
      " |-- london: double (nullable = true)\n",
      " |-- long_beach: double (nullable = true)\n",
      " |-- los_angeles: double (nullable = true)\n",
      " |-- louisiana: double (nullable = true)\n",
      " |-- louisville: double (nullable = true)\n",
      " |-- low_cal: double (nullable = true)\n",
      " |-- low_carb: double (nullable = true)\n",
      " |-- low_cholesterol: double (nullable = true)\n",
      " |-- low_fat: double (nullable = true)\n",
      " |-- low_sodium: double (nullable = true)\n",
      " |-- low_sugar: double (nullable = true)\n",
      " |-- low_no_sugar: double (nullable = true)\n",
      " |-- lunar_new_year: double (nullable = true)\n",
      " |-- lunch: double (nullable = true)\n",
      " |-- lychee: double (nullable = true)\n",
      " |-- macadamia_nut: double (nullable = true)\n",
      " |-- macaroni_and_cheese: double (nullable = true)\n",
      " |-- maine: double (nullable = true)\n",
      " |-- mandoline: double (nullable = true)\n",
      " |-- mango: double (nullable = true)\n",
      " |-- maple_syrup: double (nullable = true)\n",
      " |-- mardi_gras: double (nullable = true)\n",
      " |-- margarita: double (nullable = true)\n",
      " |-- marinade: double (nullable = true)\n",
      " |-- marinate: double (nullable = true)\n",
      " |-- marsala: double (nullable = true)\n",
      " |-- marscarpone: double (nullable = true)\n",
      " |-- marshmallow: double (nullable = true)\n",
      " |-- martini: double (nullable = true)\n",
      " |-- maryland: double (nullable = true)\n",
      " |-- massachusetts: double (nullable = true)\n",
      " |-- mayonnaise: double (nullable = true)\n",
      " |-- meat: double (nullable = true)\n",
      " |-- meatball: double (nullable = true)\n",
      " |-- meatloaf: double (nullable = true)\n",
      " |-- melon: double (nullable = true)\n",
      " |-- mexico: double (nullable = true)\n",
      " |-- mezcal: double (nullable = true)\n",
      " |-- miami: double (nullable = true)\n",
      " |-- michigan: double (nullable = true)\n",
      " |-- microwave: double (nullable = true)\n",
      " |-- midori: double (nullable = true)\n",
      " |-- milk_cream: double (nullable = true)\n",
      " |-- minneapolis: double (nullable = true)\n",
      " |-- minnesota: double (nullable = true)\n",
      " |-- mint: double (nullable = true)\n",
      " |-- mississippi: double (nullable = true)\n",
      " |-- missouri: double (nullable = true)\n",
      " |-- mixer: double (nullable = true)\n",
      " |-- molasses: double (nullable = true)\n",
      " |-- monterey_jack: double (nullable = true)\n",
      " |-- mortar_and_pestle: double (nullable = true)\n",
      " |-- mothers_day: double (nullable = true)\n",
      " |-- mozzarella: double (nullable = true)\n",
      " |-- muffin: double (nullable = true)\n",
      " |-- mushroom: double (nullable = true)\n",
      " |-- mussel: double (nullable = true)\n",
      " |-- mustard: double (nullable = true)\n",
      " |-- mustard_greens: double (nullable = true)\n",
      " |-- nancy_silverton: double (nullable = true)\n",
      " |-- nebraska: double (nullable = true)\n",
      " |-- nectarine: double (nullable = true)\n",
      " |-- new_hampshire: double (nullable = true)\n",
      " |-- new_jersey: double (nullable = true)\n",
      " |-- new_mexico: double (nullable = true)\n",
      " |-- new_orleans: double (nullable = true)\n",
      " |-- new_years_day: double (nullable = true)\n",
      " |-- new_years_eve: double (nullable = true)\n",
      " |-- new_york: double (nullable = true)\n",
      " |-- no_meat_no_problem: double (nullable = true)\n",
      " |-- no_sugar_added: double (nullable = true)\n",
      " |-- no_cook: double (nullable = true)\n",
      " |-- non_alcoholic: double (nullable = true)\n",
      " |-- noodle: double (nullable = true)\n",
      " |-- north_carolina: double (nullable = true)\n",
      " |-- nut: double (nullable = true)\n",
      " |-- nutmeg: double (nullable = true)\n",
      " |-- oat: double (nullable = true)\n",
      " |-- oatmeal: double (nullable = true)\n",
      " |-- octopus: double (nullable = true)\n",
      " |-- ohio: double (nullable = true)\n",
      " |-- oklahoma: double (nullable = true)\n",
      " |-- okra: double (nullable = true)\n",
      " |-- oktoberfest: double (nullable = true)\n",
      " |-- olive: double (nullable = true)\n",
      " |-- omelet: double (nullable = true)\n",
      " |-- one_pot_meal: double (nullable = true)\n",
      " |-- onion: double (nullable = true)\n",
      " |-- orange: double (nullable = true)\n",
      " |-- orange_juice: double (nullable = true)\n",
      " |-- oregano: double (nullable = true)\n",
      " |-- oregon: double (nullable = true)\n",
      " |-- organic: double (nullable = true)\n",
      " |-- orzo: double (nullable = true)\n",
      " |-- oscars: double (nullable = true)\n",
      " |-- oyster: double (nullable = true)\n",
      " |-- pacific_palisades: double (nullable = true)\n",
      " |-- paleo: double (nullable = true)\n",
      " |-- pan_fry: double (nullable = true)\n",
      " |-- pancake: double (nullable = true)\n",
      " |-- papaya: double (nullable = true)\n",
      " |-- paprika: double (nullable = true)\n",
      " |-- parade: double (nullable = true)\n",
      " |-- paris: double (nullable = true)\n",
      " |-- parmesan: double (nullable = true)\n",
      " |-- parsley: double (nullable = true)\n",
      " |-- parsnip: double (nullable = true)\n",
      " |-- party: double (nullable = true)\n",
      " |-- pasadena: double (nullable = true)\n",
      " |-- passion_fruit: double (nullable = true)\n",
      " |-- passover: double (nullable = true)\n",
      " |-- pasta: double (nullable = true)\n",
      " |-- pasta_maker: double (nullable = true)\n",
      " |-- pastry: double (nullable = true)\n",
      " |-- pea: double (nullable = true)\n",
      " |-- peach: double (nullable = true)\n",
      " |-- peanut: double (nullable = true)\n",
      " |-- peanut_butter: double (nullable = true)\n",
      " |-- peanut_free: double (nullable = true)\n",
      " |-- pear: double (nullable = true)\n",
      " |-- pecan: double (nullable = true)\n",
      " |-- pennsylvania: double (nullable = true)\n",
      " |-- pepper: double (nullable = true)\n",
      " |-- pernod: double (nullable = true)\n",
      " |-- persian_new_year: double (nullable = true)\n",
      " |-- persimmon: double (nullable = true)\n",
      " |-- peru: double (nullable = true)\n",
      " |-- pescatarian: double (nullable = true)\n",
      " |-- philippines: double (nullable = true)\n",
      " |-- phyllo_puff_pastry_dough: double (nullable = true)\n",
      " |-- pickles: double (nullable = true)\n",
      " |-- picnic: double (nullable = true)\n",
      " |-- pie: double (nullable = true)\n",
      " |-- pine_nut: double (nullable = true)\n",
      " |-- pineapple: double (nullable = true)\n",
      " |-- pistachio: double (nullable = true)\n",
      " |-- pittsburgh: double (nullable = true)\n",
      " |-- pizza: double (nullable = true)\n",
      " |-- plantain: double (nullable = true)\n",
      " |-- plum: double (nullable = true)\n",
      " |-- poach: double (nullable = true)\n",
      " |-- poblano: double (nullable = true)\n",
      " |-- poker_game_night: double (nullable = true)\n",
      " |-- pomegranate: double (nullable = true)\n",
      " |-- pomegranate_juice: double (nullable = true)\n",
      " |-- poppy: double (nullable = true)\n",
      " |-- pork: double (nullable = true)\n",
      " |-- pork_chop: double (nullable = true)\n",
      " |-- pork_rib: double (nullable = true)\n",
      " |-- pork_tenderloin: double (nullable = true)\n",
      " |-- port: double (nullable = true)\n",
      " |-- portland: double (nullable = true)\n",
      " |-- pot_pie: double (nullable = true)\n",
      " |-- potato: double (nullable = true)\n",
      " |-- potato_salad: double (nullable = true)\n",
      " |-- potluck: double (nullable = true)\n",
      " |-- poultry: double (nullable = true)\n",
      " |-- poultry_sausage: double (nullable = true)\n",
      " |-- pressure_cooker: double (nullable = true)\n",
      " |-- prosciutto: double (nullable = true)\n",
      " |-- providence: double (nullable = true)\n",
      " |-- prune: double (nullable = true)\n",
      " |-- pumpkin: double (nullable = true)\n",
      " |-- punch: double (nullable = true)\n",
      " |-- purim: double (nullable = true)\n",
      " |-- quail: double (nullable = true)\n",
      " |-- quiche: double (nullable = true)\n",
      " |-- quick_and_easy: double (nullable = true)\n",
      " |-- quick_and_healthy: double (nullable = true)\n",
      " |-- quince: double (nullable = true)\n",
      " |-- quinoa: double (nullable = true)\n",
      " |-- rabbit: double (nullable = true)\n",
      " |-- rack_of_lamb: double (nullable = true)\n",
      " |-- radicchio: double (nullable = true)\n",
      " |-- radish: double (nullable = true)\n",
      " |-- raisin: double (nullable = true)\n",
      " |-- ramadan: double (nullable = true)\n",
      " |-- ramekin: double (nullable = true)\n",
      " |-- raspberry: double (nullable = true)\n",
      " |-- raw: double (nullable = true)\n",
      " |-- red_wine: double (nullable = true)\n",
      " |-- rhode_island: double (nullable = true)\n",
      " |-- rhubarb: double (nullable = true)\n",
      " |-- rice: double (nullable = true)\n",
      " |-- ricotta: double (nullable = true)\n",
      " |-- roast: double (nullable = true)\n",
      " |-- root_vegetable: double (nullable = true)\n",
      " |-- rosemary: double (nullable = true)\n",
      " |-- rosh_hashanah_yom_kippur: double (nullable = true)\n",
      " |-- rosé: double (nullable = true)\n",
      " |-- rub: double (nullable = true)\n",
      " |-- rum: double (nullable = true)\n",
      " |-- rutabaga: double (nullable = true)\n",
      " |-- rye: double (nullable = true)\n",
      " |-- saffron: double (nullable = true)\n",
      " |-- sage: double (nullable = true)\n",
      " |-- sake: double (nullable = true)\n",
      " |-- salad: double (nullable = true)\n",
      " |-- salad_dressing: double (nullable = true)\n",
      " |-- salmon: double (nullable = true)\n",
      " |-- salsa: double (nullable = true)\n",
      " |-- san_francisco: double (nullable = true)\n",
      " |-- sandwich: double (nullable = true)\n",
      " |-- sandwich_theory: double (nullable = true)\n",
      " |-- sangria: double (nullable = true)\n",
      " |-- santa_monica: double (nullable = true)\n",
      " |-- sardine: double (nullable = true)\n",
      " |-- sauce: double (nullable = true)\n",
      " |-- sausage: double (nullable = true)\n",
      " |-- sauté: double (nullable = true)\n",
      " |-- scallop: double (nullable = true)\n",
      " |-- scotch: double (nullable = true)\n",
      " |-- seafood: double (nullable = true)\n",
      " |-- seattle: double (nullable = true)\n",
      " |-- seed: double (nullable = true)\n",
      " |-- self: double (nullable = true)\n",
      " |-- semolina: double (nullable = true)\n",
      " |-- sesame: double (nullable = true)\n",
      " |-- sesame_oil: double (nullable = true)\n",
      " |-- shallot: double (nullable = true)\n",
      " |-- shavuot: double (nullable = true)\n",
      " |-- shellfish: double (nullable = true)\n",
      " |-- sherry: double (nullable = true)\n",
      " |-- shower: double (nullable = true)\n",
      " |-- shrimp: double (nullable = true)\n",
      " |-- side: double (nullable = true)\n",
      " |-- simmer: double (nullable = true)\n",
      " |-- skewer: double (nullable = true)\n",
      " |-- slow_cooker: double (nullable = true)\n",
      " |-- smoker: double (nullable = true)\n",
      " |-- smoothie: double (nullable = true)\n",
      " |-- snapper: double (nullable = true)\n",
      " |-- sorbet: double (nullable = true)\n",
      " |-- soufflé_meringue: double (nullable = true)\n",
      " |-- soup_stew: double (nullable = true)\n",
      " |-- sour_cream: double (nullable = true)\n",
      " |-- sourdough: double (nullable = true)\n",
      " |-- south_carolina: double (nullable = true)\n",
      " |-- soy: double (nullable = true)\n",
      " |-- soy_free: double (nullable = true)\n",
      " |-- soy_sauce: double (nullable = true)\n",
      " |-- spain: double (nullable = true)\n",
      " |-- sparkling_wine: double (nullable = true)\n",
      " |-- spice: double (nullable = true)\n",
      " |-- spinach: double (nullable = true)\n",
      " |-- spirit: double (nullable = true)\n",
      " |-- spring: double (nullable = true)\n",
      " |-- spritzer: double (nullable = true)\n",
      " |-- squash: double (nullable = true)\n",
      " |-- squid: double (nullable = true)\n",
      " |-- st_louis: double (nullable = true)\n",
      " |-- st_patricks_day: double (nullable = true)\n",
      " |-- steak: double (nullable = true)\n",
      " |-- steam: double (nullable = true)\n",
      " |-- stew: double (nullable = true)\n",
      " |-- stir_fry: double (nullable = true)\n",
      " |-- stock: double (nullable = true)\n",
      " |-- strawberry: double (nullable = true)\n",
      " |-- stuffing_dressing: double (nullable = true)\n",
      " |-- sugar_conscious: double (nullable = true)\n",
      " |-- sugar_snap_pea: double (nullable = true)\n",
      " |-- sukkot: double (nullable = true)\n",
      " |-- summer: double (nullable = true)\n",
      " |-- super_bowl: double (nullable = true)\n",
      " |-- suzanne_goin: double (nullable = true)\n",
      " |-- sweet_potato_yam: double (nullable = true)\n",
      " |-- swiss_cheese: double (nullable = true)\n",
      " |-- switzerland: double (nullable = true)\n",
      " |-- swordfish: double (nullable = true)\n",
      " |-- taco: double (nullable = true)\n",
      " |-- tailgating: double (nullable = true)\n",
      " |-- tamarind: double (nullable = true)\n",
      " |-- tangerine: double (nullable = true)\n",
      " |-- tapioca: double (nullable = true)\n",
      " |-- tarragon: double (nullable = true)\n",
      " |-- tart: double (nullable = true)\n",
      " |-- tea: double (nullable = true)\n",
      " |-- tennessee: double (nullable = true)\n",
      " |-- tequila: double (nullable = true)\n",
      " |-- tested_and_improved: double (nullable = true)\n",
      " |-- texas: double (nullable = true)\n",
      " |-- thanksgiving: double (nullable = true)\n",
      " |-- thyme: double (nullable = true)\n",
      " |-- tilapia: double (nullable = true)\n",
      " |-- tofu: double (nullable = true)\n",
      " |-- tomatillo: double (nullable = true)\n",
      " |-- tomato: double (nullable = true)\n",
      " |-- tortillas: double (nullable = true)\n",
      " |-- tree_nut: double (nullable = true)\n",
      " |-- tree_nut_free: double (nullable = true)\n",
      " |-- triple_sec: double (nullable = true)\n",
      " |-- tropical_fruit: double (nullable = true)\n",
      " |-- trout: double (nullable = true)\n",
      " |-- tuna: double (nullable = true)\n",
      " |-- turnip: double (nullable = true)\n",
      " |-- utah: double (nullable = true)\n",
      " |-- valentines_day: double (nullable = true)\n",
      " |-- vanilla: double (nullable = true)\n",
      " |-- veal: double (nullable = true)\n",
      " |-- vegan: double (nullable = true)\n",
      " |-- vegetable: double (nullable = true)\n",
      " |-- vegetarian: double (nullable = true)\n",
      " |-- venison: double (nullable = true)\n",
      " |-- vermont: double (nullable = true)\n",
      " |-- vermouth: double (nullable = true)\n",
      " |-- vinegar: double (nullable = true)\n",
      " |-- virginia: double (nullable = true)\n",
      " |-- vodka: double (nullable = true)\n",
      " |-- waffle: double (nullable = true)\n",
      " |-- walnut: double (nullable = true)\n",
      " |-- wasabi: double (nullable = true)\n",
      " |-- washington: double (nullable = true)\n",
      " |-- washington_dc: double (nullable = true)\n",
      " |-- watercress: double (nullable = true)\n",
      " |-- watermelon: double (nullable = true)\n",
      " |-- wedding: double (nullable = true)\n",
      " |-- weelicious: double (nullable = true)\n",
      " |-- west_virginia: double (nullable = true)\n",
      " |-- westwood: double (nullable = true)\n",
      " |-- wheat_gluten_free: double (nullable = true)\n",
      " |-- whiskey: double (nullable = true)\n",
      " |-- white_wine: double (nullable = true)\n",
      " |-- whole_wheat: double (nullable = true)\n",
      " |-- wild_rice: double (nullable = true)\n",
      " |-- windsor: double (nullable = true)\n",
      " |-- wine: double (nullable = true)\n",
      " |-- winter: double (nullable = true)\n",
      " |-- wisconsin: double (nullable = true)\n",
      " |-- wok: double (nullable = true)\n",
      " |-- yellow_squash: double (nullable = true)\n",
      " |-- yogurt: double (nullable = true)\n",
      " |-- yonkers: double (nullable = true)\n",
      " |-- yuca: double (nullable = true)\n",
      " |-- zucchini: double (nullable = true)\n",
      " |-- cookbooks: double (nullable = true)\n",
      " |-- leftovers: double (nullable = true)\n",
      " |-- snack: double (nullable = true)\n",
      " |-- snack_week: double (nullable = true)\n",
      " |-- turkey: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food = food.toDF(*[sanitize_column_name(col) for col in food.columns])\n",
    "food.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb4f3d3-7ccb-4595-9852-95d5aa54f15c",
   "metadata": {},
   "source": [
    "`nrows:20057, ncols:680`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4caf6ece-3841-478c-9548-6c15d408284b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---------------------------------------------------------------------------------+------------------+\n",
      "|summary|title                                                                            |protein           |\n",
      "+-------+---------------------------------------------------------------------------------+------------------+\n",
      "|count  |20057                                                                            |15891             |\n",
      "|mean   |NULL                                                                             |100.19911427852243|\n",
      "|stddev |NULL                                                                             |3840.199434541589 |\n",
      "|min    |                Blistered-Chile–Pumpkin Seed Salsa \"                             |0.0               |\n",
      "|25%    |NULL                                                                             |3.0               |\n",
      "|50%    |NULL                                                                             |8.0               |\n",
      "|75%    |NULL                                                                             |27.0              |\n",
      "|max    |Zuppa di Cavolo Nero, Cannellini, e Salsicce: Kale, White Bean, and Sausage Soup |236489.0          |\n",
      "+-------+---------------------------------------------------------------------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.select([\"title\",\"protein\"]).summary().show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "567c9a6f-8b3c-4672-abf2-dd449f37a292",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# help(food.agg)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "239e0025-ebeb-4f66-abd7-5545cc1205b7",
   "metadata": {},
   "source": [
    "感觉`agg`从向量转换成标量就不管是否有`groupby`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "971ff78f-2c00-44fa-b5f6-8d9788ec0170",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title                     0    False\n",
       "rating                    0    False\n",
       "calories                  0    False\n",
       "protein                   0    False\n",
       "fat                       0    False\n",
       "sodium                    0    False\n",
       "cakeweek                  0    False\n",
       "wasteless                 0    False\n",
       "22_minute_meals           0     True\n",
       "3_ingredient_recipes      0     True\n",
       "30_days_of_groceries      0     True\n",
       "advance_prep_required     0     True\n",
       "alabama                   0     True\n",
       "alaska                    0     True\n",
       "alcoholic                 0     True\n",
       "almond                    0     True\n",
       "amaretto                  0     True\n",
       "anchovy                   0     True\n",
       "anise                     0     True\n",
       "anniversary               0     True\n",
       "anthony_bourdain          0     True\n",
       "aperitif                  0     True\n",
       "appetizer                 0     True\n",
       "apple                     0     True\n",
       "apple_juice               0     True\n",
       "apricot                   0     True\n",
       "arizona                   0     True\n",
       "artichoke                 0     True\n",
       "arugula                   0     True\n",
       "asian_pear                0     True\n",
       "asparagus                 0     True\n",
       "aspen                     0     True\n",
       "atlanta                   0     True\n",
       "australia                 0     True\n",
       "avocado                   0     True\n",
       "back_to_school            0     True\n",
       "backyard_bbq              0     True\n",
       "bacon                     0     True\n",
       "bake                      0     True\n",
       "banana                    0     True\n",
       "barley                    0     True\n",
       "basil                     0     True\n",
       "bass                      0     True\n",
       "bastille_day              0     True\n",
       "bean                      0     True\n",
       "beef                      0     True\n",
       "beef_rib                  0     True\n",
       "beef_shank                0     True\n",
       "beef_tenderloin           0     True\n",
       "beer                      0     True\n",
       "beet                      0     True\n",
       "bell_pepper               0     True\n",
       "berry                     0     True\n",
       "beverly_hills             0     True\n",
       "birthday                  0     True\n",
       "biscuit                   0     True\n",
       "bitters                   0     True\n",
       "blackberry                0     True\n",
       "blender                   0     True\n",
       "blue_cheese               0     True\n",
       "blueberry                 0     True\n",
       "boil                      0     True\n",
       "bok_choy                  0     True\n",
       "bon_appétit               0     True\n",
       "bon_apptit                0     True\n",
       "boston                    0     True\n",
       "bourbon                   0     True\n",
       "braise                    0     True\n",
       "bran                      0     True\n",
       "brandy                    0     True\n",
       "bread                     0     True\n",
       "breadcrumbs               0     True\n",
       "breakfast                 0     True\n",
       "brie                      0     True\n",
       "brine                     0     True\n",
       "brisket                   0     True\n",
       "broccoli                  0     True\n",
       "broccoli_rabe             0     True\n",
       "broil                     0     True\n",
       "brooklyn                  0     True\n",
       "brown_rice                0     True\n",
       "brownie                   0     True\n",
       "brunch                    0     True\n",
       "brussel_sprout            0     True\n",
       "buffalo                   0     True\n",
       "buffet                    0     True\n",
       "bulgaria                  0     True\n",
       "bulgur                    0     True\n",
       "burrito                   0     True\n",
       "butter                    0     True\n",
       "buttermilk                0     True\n",
       "butternut_squash          0     True\n",
       "butterscotch_caramel      0     True\n",
       "cabbage                   0     True\n",
       "cake                      0     True\n",
       "california                0     True\n",
       "calvados                  0     True\n",
       "cambridge                 0     True\n",
       "campari                   0     True\n",
       "camping                   0     True\n",
       "canada                    0     True\n",
       "candy                     0     True\n",
       "candy_thermometer         0     True\n",
       "cantaloupe                0     True\n",
       "capers                    0     True\n",
       "caraway                   0     True\n",
       "cardamom                  0     True\n",
       "carrot                    0     True\n",
       "cashew                    0     True\n",
       "casserole_gratin          0     True\n",
       "cauliflower               0     True\n",
       "caviar                    0     True\n",
       "celery                    0     True\n",
       "chambord                  0     True\n",
       "champagne                 0     True\n",
       "chard                     0     True\n",
       "chartreuse                0     True\n",
       "cheddar                   0     True\n",
       "cheese                    0     True\n",
       "cherry                    0     True\n",
       "chestnut                  0     True\n",
       "chicago                   0     True\n",
       "chicken                   0     True\n",
       "chickpea                  0     True\n",
       "chile                     0     True\n",
       "chile_pepper              0     True\n",
       "chili                     0     True\n",
       "chill                     0     True\n",
       "chive                     0     True\n",
       "chocolate                 0     True\n",
       "christmas                 0     True\n",
       "christmas_eve             0     True\n",
       "cilantro                  0     True\n",
       "cinco_de_mayo             0     True\n",
       "cinnamon                  0     True\n",
       "citrus                    0     True\n",
       "clam                      0     True\n",
       "clove                     0     True\n",
       "cobbler_crumble           0     True\n",
       "cocktail                  0     True\n",
       "cocktail_party            0     True\n",
       "coconut                   0     True\n",
       "cod                       0     True\n",
       "coffee                    0     True\n",
       "coffee_grinder            0     True\n",
       "cognac_armagnac           0     True\n",
       "collard_greens            0     True\n",
       "colorado                  0     True\n",
       "columbus                  0     True\n",
       "condiment                 0     True\n",
       "condiment_spread          0     True\n",
       "connecticut               0     True\n",
       "cook_like_a_diner         0     True\n",
       "cookbook_critic           0     True\n",
       "cookie                    0     True\n",
       "cookies                   0     True\n",
       "coriander                 0     True\n",
       "corn                      0     True\n",
       "cornmeal                  0     True\n",
       "costa_mesa                0     True\n",
       "cottage_cheese            0     True\n",
       "couscous                  0     True\n",
       "crab                      0     True\n",
       "cranberry                 0     True\n",
       "cranberry_sauce           0     True\n",
       "cream_cheese              0     True\n",
       "créme_de_cacao            0     True\n",
       "crêpe                     0     True\n",
       "crme_de_cacao             0     True\n",
       "cuba                      0     True\n",
       "cucumber                  0     True\n",
       "cumin                     0     True\n",
       "cupcake                   0     True\n",
       "currant                   0     True\n",
       "curry                     0     True\n",
       "custard                   0     True\n",
       "dairy                     0     True\n",
       "dairy_free                0     True\n",
       "dallas                    0     True\n",
       "date                      0     True\n",
       "deep_fry                  0     True\n",
       "denver                    0     True\n",
       "dessert                   0     True\n",
       "digestif                  0     True\n",
       "dill                      0     True\n",
       "dinner                    0     True\n",
       "dip                       0     True\n",
       "diwali                    0     True\n",
       "dominican_republic        0     True\n",
       "dorie_greenspan           0     True\n",
       "double_boiler             0     True\n",
       "dried_fruit               0     True\n",
       "drink                     0     True\n",
       "drinks                    0     True\n",
       "duck                      0     True\n",
       "easter                    0     True\n",
       "eau_de_vie                0     True\n",
       "edible_gift               0     True\n",
       "egg                       0     True\n",
       "egg_nog                   0     True\n",
       "eggplant                  0     True\n",
       "egypt                     0     True\n",
       "emeril_lagasse            0     True\n",
       "endive                    0     True\n",
       "engagement_party          0     True\n",
       "england                   0     True\n",
       "entertaining              0     True\n",
       "epi__ushg                 0     True\n",
       "epi_loves_the_microwave   0     True\n",
       "escarole                  0     True\n",
       "fall                      0     True\n",
       "family_reunion            0     True\n",
       "fat_free                  0     True\n",
       "fathers_day               0     True\n",
       "fennel                    0     True\n",
       "feta                      0     True\n",
       "fig                       0     True\n",
       "fish                      0     True\n",
       "flaming_hot_summer        0     True\n",
       "flat_bread                0     True\n",
       "florida                   0     True\n",
       "fontina                   0     True\n",
       "food_processor            0     True\n",
       "fortified_wine            0     True\n",
       "fourth_of_july            0     True\n",
       "france                    0     True\n",
       "frangelico                0     True\n",
       "frankenrecipe             0     True\n",
       "freeze_chill              0     True\n",
       "freezer_food              0     True\n",
       "friendsgiving             0     True\n",
       "frittata                  0     True\n",
       "fritter                   0     True\n",
       "frozen_dessert            0     True\n",
       "fruit                     0     True\n",
       "fruit_juice               0     True\n",
       "fry                       0     True\n",
       "game                      0     True\n",
       "garlic                    0     True\n",
       "georgia                   0     True\n",
       "germany                   0     True\n",
       "gin                       0     True\n",
       "ginger                    0     True\n",
       "goat_cheese               0     True\n",
       "goose                     0     True\n",
       "gouda                     0     True\n",
       "gourmet                   0     True\n",
       "graduation                0     True\n",
       "grains                    0     True\n",
       "grand_marnier             0     True\n",
       "granola                   0     True\n",
       "grape                     0     True\n",
       "grapefruit                0     True\n",
       "grappa                    0     True\n",
       "green_bean                0     True\n",
       "green_onion_scallion      0     True\n",
       "grill                     0     True\n",
       "grill_barbecue            0     True\n",
       "ground_beef               0     True\n",
       "ground_lamb               0     True\n",
       "guam                      0     True\n",
       "guava                     0     True\n",
       "haiti                     0     True\n",
       "halibut                   0     True\n",
       "halloween                 0     True\n",
       "ham                       0     True\n",
       "hamburger                 0     True\n",
       "hanukkah                  0     True\n",
       "harpercollins             0     True\n",
       "hawaii                    0     True\n",
       "hazelnut                  0     True\n",
       "healdsburg                0     True\n",
       "healthy                   0     True\n",
       "herb                      0     True\n",
       "high_fiber                0     True\n",
       "hollywood                 0     True\n",
       "hominy_cornmeal_masa      0     True\n",
       "honey                     0     True\n",
       "honeydew                  0     True\n",
       "hors_doeuvre              0     True\n",
       "horseradish               0     True\n",
       "hot_drink                 0     True\n",
       "hot_pepper                0     True\n",
       "house_and_garden          0     True\n",
       "house_cocktail            0     True\n",
       "houston                   0     True\n",
       "hummus                    0     True\n",
       "ice_cream                 0     True\n",
       "ice_cream_machine         0     True\n",
       "iced_coffee               0     True\n",
       "iced_tea                  0     True\n",
       "idaho                     0     True\n",
       "illinois                  0     True\n",
       "indiana                   0     True\n",
       "iowa                      0     True\n",
       "ireland                   0     True\n",
       "israel                    0     True\n",
       "italy                     0     True\n",
       "jalapeño                  0     True\n",
       "jam_or_jelly              0     True\n",
       "jamaica                   0     True\n",
       "japan                     0     True\n",
       "jerusalem_artichoke       0     True\n",
       "juicer                    0     True\n",
       "jícama                    0     True\n",
       "kahlúa                    0     True\n",
       "kale                      0     True\n",
       "kansas                    0     True\n",
       "kansas_city               0     True\n",
       "kentucky                  0     True\n",
       "kentucky_derby            0     True\n",
       "kid_friendly              0     True\n",
       "kidney_friendly           0     True\n",
       "kirsch                    0     True\n",
       "kitchen_olympics          0     True\n",
       "kiwi                      0     True\n",
       "kosher                    0     True\n",
       "kosher_for_passover       0     True\n",
       "kumquat                   0     True\n",
       "kwanzaa                   0     True\n",
       "labor_day                 0     True\n",
       "lamb                      0     True\n",
       "lamb_chop                 0     True\n",
       "lamb_shank                0     True\n",
       "lancaster                 0     True\n",
       "las_vegas                 0     True\n",
       "lasagna                   0     True\n",
       "leafy_green               0     True\n",
       "leek                      0     True\n",
       "legume                    0     True\n",
       "lemon                     0     True\n",
       "lemon_juice               0     True\n",
       "lemongrass                0     True\n",
       "lentil                    0     True\n",
       "lettuce                   0     True\n",
       "lima_bean                 0     True\n",
       "lime                      0     True\n",
       "lime_juice                0     True\n",
       "lingonberry               0     True\n",
       "liqueur                   0     True\n",
       "lobster                   0     True\n",
       "london                    0     True\n",
       "long_beach                0     True\n",
       "los_angeles               0     True\n",
       "louisiana                 0     True\n",
       "louisville                0     True\n",
       "low_cal                   0     True\n",
       "low_carb                  0     True\n",
       "low_cholesterol           0     True\n",
       "low_fat                   0     True\n",
       "low_sodium                0     True\n",
       "low_sugar                 0     True\n",
       "low_no_sugar              0     True\n",
       "lunar_new_year            0     True\n",
       "lunch                     0     True\n",
       "lychee                    0     True\n",
       "macadamia_nut             0     True\n",
       "macaroni_and_cheese       0     True\n",
       "maine                     0     True\n",
       "mandoline                 0     True\n",
       "mango                     0     True\n",
       "maple_syrup               0     True\n",
       "mardi_gras                0     True\n",
       "margarita                 0     True\n",
       "marinade                  0     True\n",
       "marinate                  0     True\n",
       "marsala                   0     True\n",
       "marscarpone               0     True\n",
       "marshmallow               0     True\n",
       "martini                   0     True\n",
       "maryland                  0     True\n",
       "massachusetts             0     True\n",
       "mayonnaise                0     True\n",
       "meat                      0     True\n",
       "meatball                  0     True\n",
       "meatloaf                  0     True\n",
       "melon                     0     True\n",
       "mexico                    0     True\n",
       "mezcal                    0     True\n",
       "miami                     0     True\n",
       "michigan                  0     True\n",
       "microwave                 0     True\n",
       "midori                    0     True\n",
       "milk_cream                0     True\n",
       "minneapolis               0     True\n",
       "minnesota                 0     True\n",
       "mint                      0     True\n",
       "mississippi               0     True\n",
       "missouri                  0     True\n",
       "mixer                     0     True\n",
       "molasses                  0     True\n",
       "monterey_jack             0     True\n",
       "mortar_and_pestle         0     True\n",
       "mothers_day               0     True\n",
       "mozzarella                0     True\n",
       "muffin                    0     True\n",
       "mushroom                  0     True\n",
       "mussel                    0     True\n",
       "mustard                   0     True\n",
       "mustard_greens            0     True\n",
       "nancy_silverton           0     True\n",
       "nebraska                  0     True\n",
       "nectarine                 0     True\n",
       "new_hampshire             0     True\n",
       "new_jersey                0     True\n",
       "new_mexico                0     True\n",
       "new_orleans               0     True\n",
       "new_years_day             0     True\n",
       "new_years_eve             0     True\n",
       "new_york                  0     True\n",
       "no_meat_no_problem        0     True\n",
       "no_sugar_added            0     True\n",
       "no_cook                   0     True\n",
       "non_alcoholic             0     True\n",
       "noodle                    0     True\n",
       "north_carolina            0     True\n",
       "nut                       0     True\n",
       "nutmeg                    0     True\n",
       "oat                       0     True\n",
       "oatmeal                   0     True\n",
       "octopus                   0     True\n",
       "ohio                      0     True\n",
       "oklahoma                  0     True\n",
       "okra                      0     True\n",
       "oktoberfest               0     True\n",
       "olive                     0     True\n",
       "omelet                    0     True\n",
       "one_pot_meal              0     True\n",
       "onion                     0     True\n",
       "orange                    0     True\n",
       "orange_juice              0     True\n",
       "oregano                   0     True\n",
       "oregon                    0     True\n",
       "organic                   0     True\n",
       "orzo                      0     True\n",
       "oscars                    0     True\n",
       "oyster                    0     True\n",
       "pacific_palisades         0     True\n",
       "paleo                     0     True\n",
       "pan_fry                   0     True\n",
       "pancake                   0     True\n",
       "papaya                    0     True\n",
       "paprika                   0     True\n",
       "parade                    0     True\n",
       "paris                     0     True\n",
       "parmesan                  0     True\n",
       "parsley                   0     True\n",
       "parsnip                   0     True\n",
       "party                     0     True\n",
       "pasadena                  0     True\n",
       "passion_fruit             0     True\n",
       "passover                  0     True\n",
       "pasta                     0     True\n",
       "pasta_maker               0     True\n",
       "pastry                    0     True\n",
       "pea                       0     True\n",
       "peach                     0     True\n",
       "peanut                    0     True\n",
       "peanut_butter             0     True\n",
       "peanut_free               0     True\n",
       "pear                      0     True\n",
       "pecan                     0     True\n",
       "pennsylvania              0     True\n",
       "pepper                    0     True\n",
       "pernod                    0     True\n",
       "persian_new_year          0     True\n",
       "persimmon                 0     True\n",
       "peru                      0     True\n",
       "pescatarian               0     True\n",
       "philippines               0     True\n",
       "phyllo_puff_pastry_dough  0     True\n",
       "pickles                   0     True\n",
       "picnic                    0     True\n",
       "pie                       0     True\n",
       "pine_nut                  0     True\n",
       "pineapple                 0     True\n",
       "pistachio                 0     True\n",
       "pittsburgh                0     True\n",
       "pizza                     0     True\n",
       "plantain                  0     True\n",
       "plum                      0     True\n",
       "poach                     0     True\n",
       "poblano                   0     True\n",
       "poker_game_night          0     True\n",
       "pomegranate               0     True\n",
       "pomegranate_juice         0     True\n",
       "poppy                     0     True\n",
       "pork                      0     True\n",
       "pork_chop                 0     True\n",
       "pork_rib                  0     True\n",
       "pork_tenderloin           0     True\n",
       "port                      0     True\n",
       "portland                  0     True\n",
       "pot_pie                   0     True\n",
       "potato                    0     True\n",
       "potato_salad              0     True\n",
       "potluck                   0     True\n",
       "poultry                   0     True\n",
       "poultry_sausage           0     True\n",
       "pressure_cooker           0     True\n",
       "prosciutto                0     True\n",
       "providence                0     True\n",
       "prune                     0     True\n",
       "pumpkin                   0     True\n",
       "punch                     0     True\n",
       "purim                     0     True\n",
       "quail                     0     True\n",
       "quiche                    0     True\n",
       "quick_and_easy            0     True\n",
       "quick_and_healthy         0     True\n",
       "quince                    0     True\n",
       "quinoa                    0     True\n",
       "rabbit                    0     True\n",
       "rack_of_lamb              0     True\n",
       "radicchio                 0     True\n",
       "radish                    0     True\n",
       "raisin                    0     True\n",
       "ramadan                   0     True\n",
       "ramekin                   0     True\n",
       "raspberry                 0     True\n",
       "raw                       0     True\n",
       "red_wine                  0     True\n",
       "rhode_island              0     True\n",
       "rhubarb                   0     True\n",
       "rice                      0     True\n",
       "ricotta                   0     True\n",
       "roast                     0     True\n",
       "root_vegetable            0     True\n",
       "rosemary                  0     True\n",
       "rosh_hashanah_yom_kippur  0     True\n",
       "rosé                      0     True\n",
       "rub                       0     True\n",
       "rum                       0     True\n",
       "rutabaga                  0     True\n",
       "rye                       0     True\n",
       "saffron                   0     True\n",
       "sage                      0     True\n",
       "sake                      0     True\n",
       "salad                     0     True\n",
       "salad_dressing            0     True\n",
       "salmon                    0     True\n",
       "salsa                     0     True\n",
       "san_francisco             0     True\n",
       "sandwich                  0     True\n",
       "sandwich_theory           0     True\n",
       "sangria                   0     True\n",
       "santa_monica              0     True\n",
       "sardine                   0     True\n",
       "sauce                     0     True\n",
       "sausage                   0     True\n",
       "sauté                     0     True\n",
       "scallop                   0     True\n",
       "scotch                    0     True\n",
       "seafood                   0     True\n",
       "seattle                   0     True\n",
       "seed                      0     True\n",
       "self                      0     True\n",
       "semolina                  0     True\n",
       "sesame                    0     True\n",
       "sesame_oil                0     True\n",
       "shallot                   0     True\n",
       "shavuot                   0     True\n",
       "shellfish                 0     True\n",
       "sherry                    0     True\n",
       "shower                    0     True\n",
       "shrimp                    0     True\n",
       "side                      0     True\n",
       "simmer                    0     True\n",
       "skewer                    0     True\n",
       "slow_cooker               0     True\n",
       "smoker                    0     True\n",
       "smoothie                  0     True\n",
       "snapper                   0     True\n",
       "sorbet                    0     True\n",
       "soufflé_meringue          0     True\n",
       "soup_stew                 0     True\n",
       "sour_cream                0     True\n",
       "sourdough                 0     True\n",
       "south_carolina            0     True\n",
       "soy                       0     True\n",
       "soy_free                  0     True\n",
       "soy_sauce                 0     True\n",
       "spain                     0     True\n",
       "sparkling_wine            0     True\n",
       "spice                     0     True\n",
       "spinach                   0     True\n",
       "spirit                    0     True\n",
       "spring                    0     True\n",
       "spritzer                  0     True\n",
       "squash                    0     True\n",
       "squid                     0     True\n",
       "st_louis                  0     True\n",
       "st_patricks_day           0     True\n",
       "steak                     0     True\n",
       "steam                     0     True\n",
       "stew                      0     True\n",
       "stir_fry                  0     True\n",
       "stock                     0     True\n",
       "strawberry                0     True\n",
       "stuffing_dressing         0     True\n",
       "sugar_conscious           0     True\n",
       "sugar_snap_pea            0     True\n",
       "sukkot                    0     True\n",
       "summer                    0     True\n",
       "super_bowl                0     True\n",
       "suzanne_goin              0     True\n",
       "sweet_potato_yam          0     True\n",
       "swiss_cheese              0     True\n",
       "switzerland               0     True\n",
       "swordfish                 0     True\n",
       "taco                      0     True\n",
       "tailgating                0     True\n",
       "tamarind                  0     True\n",
       "tangerine                 0     True\n",
       "tapioca                   0     True\n",
       "tarragon                  0     True\n",
       "tart                      0     True\n",
       "tea                       0     True\n",
       "tennessee                 0     True\n",
       "tequila                   0     True\n",
       "tested_and_improved       0     True\n",
       "texas                     0     True\n",
       "thanksgiving              0     True\n",
       "thyme                     0     True\n",
       "tilapia                   0     True\n",
       "tofu                      0     True\n",
       "tomatillo                 0     True\n",
       "tomato                    0     True\n",
       "tortillas                 0     True\n",
       "tree_nut                  0     True\n",
       "tree_nut_free             0     True\n",
       "triple_sec                0     True\n",
       "tropical_fruit            0     True\n",
       "trout                     0     True\n",
       "tuna                      0     True\n",
       "turnip                    0     True\n",
       "utah                      0     True\n",
       "valentines_day            0     True\n",
       "vanilla                   0     True\n",
       "veal                      0     True\n",
       "vegan                     0     True\n",
       "vegetable                 0     True\n",
       "vegetarian                0     True\n",
       "venison                   0     True\n",
       "vermont                   0     True\n",
       "vermouth                  0     True\n",
       "vinegar                   0     True\n",
       "virginia                  0     True\n",
       "vodka                     0     True\n",
       "waffle                    0     True\n",
       "walnut                    0     True\n",
       "wasabi                    0     True\n",
       "washington                0     True\n",
       "washington_dc             0     True\n",
       "watercress                0     True\n",
       "watermelon                0     True\n",
       "wedding                   0     True\n",
       "weelicious                0     True\n",
       "west_virginia             0     True\n",
       "westwood                  0     True\n",
       "wheat_gluten_free         0     True\n",
       "whiskey                   0     True\n",
       "white_wine                0     True\n",
       "whole_wheat               0     True\n",
       "wild_rice                 0     True\n",
       "windsor                   0     True\n",
       "wine                      0     True\n",
       "winter                    0     True\n",
       "wisconsin                 0     True\n",
       "wok                       0     True\n",
       "yellow_squash             0     True\n",
       "yogurt                    0     True\n",
       "yonkers                   0     True\n",
       "yuca                      0     True\n",
       "zucchini                  0     True\n",
       "cookbooks                 0     True\n",
       "leftovers                 0     True\n",
       "snack                     0     True\n",
       "snack_week                0     True\n",
       "turkey                    0     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option(\"display.max_row\",1000)\n",
    "\n",
    "is_binary = food.agg(*list(map(lambda x:(F.size(F.collect_set(x))==2).alias(x),food.columns))).toPandas()\n",
    "is_binary.unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f938bba2-b591-4590-85e1-8c09f6c364f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+--------+-------+----+------+--------+---------+\n",
      "|               title|rating|calories|protein| fat|sodium|cakeweek|wasteless|\n",
      "+--------------------+------+--------+-------+----+------+--------+---------+\n",
      "|Lentil, Apple, an...|   2.5|   426.0|   30.0| 7.0| 559.0|     0.0|      0.0|\n",
      "|Boudin Blanc Terr...| 4.375|   403.0|   18.0|23.0|1439.0|     0.0|      0.0|\n",
      "|Potato and Fennel...|  3.75|   165.0|    6.0| 7.0| 165.0|     0.0|      0.0|\n",
      "|Mahi-Mahi in Toma...|   5.0|    NULL|   NULL|NULL|  NULL|     0.0|      0.0|\n",
      "|Spinach Noodle Ca...| 3.125|   547.0|   20.0|32.0| 452.0|     0.0|      0.0|\n",
      "+--------------------+------+--------+-------+----+------+--------+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "not_binary_cols = [\"title\",\"rating\",\"calories\",\"protein\",\"fat\",\"sodium\" ,\"cakeweek\",\"wasteless\"]\n",
    "food.select(not_binary_cols).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8ae37726-bcf1-46f5-a8e2-98645631a62f",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_df = food.select(not_binary_cols).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bff64cce-97a2-4744-92af-49922102e5ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "title                                                                            \n",
      "Pastry Dough                                                                         28\n",
      "Chicken Stock                                                                        26\n",
      "Turkey Giblet Stock                                                                  20\n",
      "Turkey Stock                                                                         15\n",
      "Simple Syrup                                                                         13\n",
      "                                                                                     ..\n",
      "Fruitcake Brownies                                                                    1\n",
      "Fruitcake Trifle                                                                      1\n",
      "Fruitcake with Ginger Cream Cheese                                                    1\n",
      "Fruited Chili Sauce                                                                   1\n",
      "Zuppa di Cavolo Nero, Cannellini, e Salsicce: Kale, White Bean, and Sausage Soup      1\n",
      "Name: count, Length: 17741, dtype: int64\n",
      "rating                  \n",
      "4.375                       8017\n",
      "3.75                        5168\n",
      "5.0                         2719\n",
      "0.0                         1836\n",
      "3.125                       1489\n",
      "2.5                          531\n",
      "1.25                         164\n",
      "1.875                        124\n",
      " Aged Balsamic Vinegar \"       1\n",
      " Aunt Gloria-Style \"           1\n",
      " Cucumber                      1\n",
      " Vermouth                      1\n",
      "Name: count, dtype: int64\n",
      "calories\n",
      "234.0       55\n",
      "228.0       45\n",
      "222.0       44\n",
      "223.0       44\n",
      "247.0       44\n",
      "            ..\n",
      "2380.0       1\n",
      "2401.0       1\n",
      "2417.0       1\n",
      "2418.0       1\n",
      "2925.0       1\n",
      "Name: count, Length: 1862, dtype: int64\n",
      "protein   \n",
      "1.000         1331\n",
      "2.000         1254\n",
      "3.000         1044\n",
      "4.000          973\n",
      "0.000          851\n",
      "5.000          828\n",
      "6.000          722\n",
      "7.000          579\n",
      "8.000          564\n",
      "9.000          440\n",
      "10.000         383\n",
      "11.000         315\n",
      "12.000         264\n",
      "13.000         244\n",
      "14.000         216\n",
      "17.000         209\n",
      "15.000         187\n",
      "16.000         183\n",
      "18.000         176\n",
      "19.000         158\n",
      "20.000         148\n",
      "21.000         143\n",
      "22.000         143\n",
      "24.000         139\n",
      "23.000         138\n",
      "35.000         133\n",
      "31.000         128\n",
      "28.000         128\n",
      "32.000         126\n",
      "29.000         121\n",
      "25.000         120\n",
      "36.000         119\n",
      "27.000         119\n",
      "30.000         115\n",
      "37.000         114\n",
      "26.000         113\n",
      "40.000         108\n",
      "34.000         104\n",
      "42.000         103\n",
      "41.000          97\n",
      "33.000          93\n",
      "38.000          90\n",
      "43.000          82\n",
      "46.000          81\n",
      "39.000          80\n",
      "45.000          80\n",
      "49.000          77\n",
      "50.000          71\n",
      "44.000          70\n",
      "48.000          68\n",
      "55.000          65\n",
      "54.000          63\n",
      "51.000          57\n",
      "47.000          54\n",
      "59.000          46\n",
      "56.000          44\n",
      "53.000          43\n",
      "60.000          41\n",
      "68.000          38\n",
      "75.000          37\n",
      "52.000          36\n",
      "64.000          34\n",
      "58.000          33\n",
      "57.000          33\n",
      "69.000          32\n",
      "65.000          31\n",
      "70.000          31\n",
      "61.000          29\n",
      "63.000          28\n",
      "81.000          28\n",
      "73.000          27\n",
      "62.000          26\n",
      "67.000          24\n",
      "74.000          24\n",
      "76.000          21\n",
      "66.000          20\n",
      "82.000          20\n",
      "85.000          19\n",
      "80.000          18\n",
      "72.000          17\n",
      "78.000          17\n",
      "79.000          17\n",
      "77.000          17\n",
      "83.000          17\n",
      "97.000          17\n",
      "71.000          16\n",
      "87.000          16\n",
      "88.000          15\n",
      "91.000          14\n",
      "100.000         14\n",
      "86.000          14\n",
      "98.000          13\n",
      "93.000          13\n",
      "90.000          13\n",
      "133.000         13\n",
      "104.000         11\n",
      "107.000         10\n",
      "89.000          10\n",
      "84.000          10\n",
      "103.000          9\n",
      "95.000           9\n",
      "106.000          9\n",
      "118.000          8\n",
      "110.000          8\n",
      "99.000           8\n",
      "94.000           8\n",
      "105.000          7\n",
      "115.000          7\n",
      "161.000          6\n",
      "129.000          6\n",
      "126.000          6\n",
      "113.000          6\n",
      "150.000          6\n",
      "171.000          6\n",
      "96.000           6\n",
      "101.000          6\n",
      "92.000           6\n",
      "140.000          6\n",
      "162.000          5\n",
      "123.000          5\n",
      "134.000          5\n",
      "175.000          5\n",
      "131.000          5\n",
      "117.000          5\n",
      "111.000          5\n",
      "167.000          5\n",
      "102.000          5\n",
      "119.000          5\n",
      "153.000          4\n",
      "142.000          4\n",
      "138.000          4\n",
      "198.000          4\n",
      "176.000          4\n",
      "152.000          4\n",
      "144.000          3\n",
      "234.000          3\n",
      "170.000          3\n",
      "157.000          3\n",
      "212.000          3\n",
      "190.000          3\n",
      "156.000          3\n",
      "159.000          3\n",
      "155.000          3\n",
      "183.000          3\n",
      "125.000          3\n",
      "108.000          3\n",
      "332.000          3\n",
      "264.000          3\n",
      "135.000          3\n",
      "112.000          3\n",
      "121.000          3\n",
      "109.000          3\n",
      "137.000          3\n",
      "189.000          2\n",
      "180.000          2\n",
      "626.000          2\n",
      "414.000          2\n",
      "192.000          2\n",
      "187.000          2\n",
      "384.000          2\n",
      "373.000          2\n",
      "940.000          2\n",
      "346.000          2\n",
      "242.000          2\n",
      "325.000          2\n",
      "312.000          2\n",
      "216.000          2\n",
      "249.000          2\n",
      "222.000          2\n",
      "226.000          2\n",
      "228.000          2\n",
      "238.000          2\n",
      "248.000          2\n",
      "338.000          2\n",
      "236489.000       2\n",
      "154.000          2\n",
      "160.000          2\n",
      "143.000          2\n",
      "145.000          2\n",
      "148.000          2\n",
      "130.000          2\n",
      "151.000          2\n",
      "114.000          2\n",
      "128.000          2\n",
      "132.000          2\n",
      "122.000          2\n",
      "120.000          2\n",
      "163.000          2\n",
      "116.000          2\n",
      "394.000          1\n",
      "439.000          1\n",
      "474.000          1\n",
      "470.000          1\n",
      "465.000          1\n",
      "451.000          1\n",
      "447.000          1\n",
      "440.000          1\n",
      "389.000          1\n",
      "421.000          1\n",
      "402.000          1\n",
      "417.000          1\n",
      "392.000          1\n",
      "393.000          1\n",
      "477.000          1\n",
      "413.000          1\n",
      "408.000          1\n",
      "406.000          1\n",
      "403.000          1\n",
      "182.000          1\n",
      "570.000          1\n",
      "493.000          1\n",
      "528.000          1\n",
      "200968.000       1\n",
      "200210.000       1\n",
      "166471.000       1\n",
      "87188.000        1\n",
      "58334.000        1\n",
      "58324.000        1\n",
      "2074.000         1\n",
      "1625.000         1\n",
      "1365.000         1\n",
      "1164.000         1\n",
      "1114.000         1\n",
      "942.000          1\n",
      "3.750            1\n",
      "907.000          1\n",
      "898.000          1\n",
      "840.000          1\n",
      "723.000          1\n",
      "658.000          1\n",
      "4.375            1\n",
      "608.000          1\n",
      "592.000          1\n",
      "378.000          1\n",
      "567.000          1\n",
      "382.000          1\n",
      "347.000          1\n",
      "377.000          1\n",
      "136.000          1\n",
      "245.000          1\n",
      "146.000          1\n",
      "147.000          1\n",
      "224.000          1\n",
      "223.000          1\n",
      "149.000          1\n",
      "217.000          1\n",
      "214.000          1\n",
      "211.000          1\n",
      "206.000          1\n",
      "203.000          1\n",
      "201.000          1\n",
      "200.000          1\n",
      "199.000          1\n",
      "196.000          1\n",
      "193.000          1\n",
      "191.000          1\n",
      "166.000          1\n",
      "173.000          1\n",
      "188.000          1\n",
      "178.000          1\n",
      "139.000          1\n",
      "250.000          1\n",
      "376.000          1\n",
      "253.000          1\n",
      "372.000          1\n",
      "359.000          1\n",
      "348.000          1\n",
      "179.000          1\n",
      "343.000          1\n",
      "124.000          1\n",
      "127.000          1\n",
      "329.000          1\n",
      "327.000          1\n",
      "321.000          1\n",
      "316.000          1\n",
      "310.000          1\n",
      "301.000          1\n",
      "297.000          1\n",
      "284.000          1\n",
      "281.000          1\n",
      "279.000          1\n",
      "270.000          1\n",
      "267.000          1\n",
      "266.000          1\n",
      "254.000          1\n",
      "141.000          1\n",
      "Name: count, dtype: int64\n",
      "fat      \n",
      "0.0          1223\n",
      "1.0           626\n",
      "14.0          465\n",
      "12.0          431\n",
      "9.0           425\n",
      "7.0           410\n",
      "11.0          399\n",
      "15.0          397\n",
      "10.0          396\n",
      "16.0          383\n",
      "13.0          378\n",
      "18.0          378\n",
      "5.0           375\n",
      "4.0           370\n",
      "6.0           364\n",
      "19.0          360\n",
      "17.0          353\n",
      "8.0           342\n",
      "21.0          333\n",
      "20.0          319\n",
      "3.0           312\n",
      "2.0           297\n",
      "22.0          293\n",
      "23.0          277\n",
      "24.0          264\n",
      "25.0          230\n",
      "27.0          224\n",
      "26.0          220\n",
      "28.0          207\n",
      "29.0          201\n",
      "30.0          199\n",
      "31.0          188\n",
      "35.0          158\n",
      "32.0          147\n",
      "33.0          144\n",
      "36.0          142\n",
      "37.0          139\n",
      "38.0          139\n",
      "34.0          135\n",
      "39.0          111\n",
      "42.0          102\n",
      "45.0          101\n",
      "43.0           98\n",
      "44.0           98\n",
      "41.0           95\n",
      "47.0           87\n",
      "48.0           82\n",
      "49.0           82\n",
      "40.0           81\n",
      "46.0           71\n",
      "52.0           70\n",
      "55.0           69\n",
      "50.0           67\n",
      "53.0           65\n",
      "51.0           63\n",
      "54.0           61\n",
      "56.0           57\n",
      "60.0           57\n",
      "57.0           54\n",
      "58.0           54\n",
      "61.0           49\n",
      "63.0           46\n",
      "73.0           43\n",
      "71.0           43\n",
      "59.0           43\n",
      "75.0           40\n",
      "68.0           40\n",
      "69.0           40\n",
      "64.0           38\n",
      "70.0           36\n",
      "65.0           35\n",
      "72.0           33\n",
      "67.0           32\n",
      "66.0           32\n",
      "76.0           32\n",
      "77.0           32\n",
      "62.0           30\n",
      "82.0           27\n",
      "78.0           26\n",
      "74.0           25\n",
      "94.0           24\n",
      "80.0           24\n",
      "79.0           22\n",
      "87.0           21\n",
      "83.0           20\n",
      "93.0           19\n",
      "91.0           17\n",
      "88.0           16\n",
      "85.0           15\n",
      "89.0           15\n",
      "90.0           15\n",
      "96.0           14\n",
      "115.0          13\n",
      "97.0           13\n",
      "103.0          13\n",
      "92.0           13\n",
      "95.0           12\n",
      "100.0          12\n",
      "101.0          12\n",
      "106.0          12\n",
      "99.0           11\n",
      "107.0          11\n",
      "109.0          11\n",
      "110.0          11\n",
      "86.0           10\n",
      "98.0           10\n",
      "102.0          10\n",
      "114.0          10\n",
      "111.0           9\n",
      "81.0            8\n",
      "112.0           8\n",
      "118.0           8\n",
      "108.0           8\n",
      "152.0           7\n",
      "120.0           7\n",
      "136.0           7\n",
      "134.0           7\n",
      "105.0           6\n",
      "104.0           6\n",
      "84.0            6\n",
      "123.0           6\n",
      "125.0           6\n",
      "142.0           6\n",
      "130.0           5\n",
      "151.0           5\n",
      "156.0           5\n",
      "135.0           5\n",
      "131.0           5\n",
      "127.0           5\n",
      "128.0           5\n",
      "166.0           5\n",
      "194.0           5\n",
      "116.0           5\n",
      "146.0           5\n",
      "161.0           4\n",
      "201.0           4\n",
      "160.0           4\n",
      "158.0           4\n",
      "170.0           4\n",
      "172.0           4\n",
      "181.0           4\n",
      "153.0           4\n",
      "163.0           4\n",
      "119.0           4\n",
      "117.0           4\n",
      "137.0           4\n",
      "132.0           4\n",
      "141.0           4\n",
      "230.0           3\n",
      "113.0           3\n",
      "193.0           3\n",
      "189.0           3\n",
      "369.0           3\n",
      "186.0           3\n",
      "182.0           3\n",
      "178.0           3\n",
      "219.0           3\n",
      "245.0           3\n",
      "165.0           3\n",
      "164.0           3\n",
      "1007.0          3\n",
      "226.0           3\n",
      "129.0           3\n",
      "133.0           3\n",
      "145.0           3\n",
      "140.0           3\n",
      "139.0           3\n",
      "148.0           3\n",
      "149.0           3\n",
      "205.0           2\n",
      "198.0           2\n",
      "199.0           2\n",
      "235.0           2\n",
      "234.0           2\n",
      "535.0           2\n",
      "204.0           2\n",
      "296.0           2\n",
      "297.0           2\n",
      "210.0           2\n",
      "292.0           2\n",
      "147.0           2\n",
      "213.0           2\n",
      "214.0           2\n",
      "446.0           2\n",
      "217.0           2\n",
      "228.0           2\n",
      "144.0           2\n",
      "192.0           2\n",
      "155.0           2\n",
      "515.0           2\n",
      "121.0           2\n",
      "382.0           2\n",
      "2228.0          2\n",
      "267.0           2\n",
      "122.0           2\n",
      "126.0           2\n",
      "124.0           2\n",
      "221495.0        2\n",
      "154.0           2\n",
      "240.0           2\n",
      "138.0           2\n",
      "188.0           2\n",
      "190.0           2\n",
      "460.0           2\n",
      "376.0           2\n",
      "459.0           1\n",
      "479.0           1\n",
      "476.0           1\n",
      "464.0           1\n",
      "456.0           1\n",
      "143.0           1\n",
      "416.0           1\n",
      "455.0           1\n",
      "449.0           1\n",
      "441.0           1\n",
      "436.0           1\n",
      "431.0           1\n",
      "422.0           1\n",
      "421.0           1\n",
      "483.0           1\n",
      "413.0           1\n",
      "398.0           1\n",
      "397.0           1\n",
      "384.0           1\n",
      "383.0           1\n",
      "360.0           1\n",
      "342.0           1\n",
      "480.0           1\n",
      "618.0           1\n",
      "493.0           1\n",
      "495.0           1\n",
      "1716279.0       1\n",
      "747374.0        1\n",
      "186660.0        1\n",
      "186642.0        1\n",
      "44198.0         1\n",
      "1818.0          1\n",
      "1610.0          1\n",
      "1480.0          1\n",
      "1115.0          1\n",
      "1054.0          1\n",
      "923.0           1\n",
      "919.0           1\n",
      "887.0           1\n",
      "776.0           1\n",
      "666.0           1\n",
      "651.0           1\n",
      "637.0           1\n",
      "633.0           1\n",
      "334.0           1\n",
      "599.0           1\n",
      "595.0           1\n",
      "563.0           1\n",
      "551.0           1\n",
      "540.0           1\n",
      "531.0           1\n",
      "519.0           1\n",
      "517.0           1\n",
      "513.0           1\n",
      "503.0           1\n",
      "336.0           1\n",
      "276.0           1\n",
      "333.0           1\n",
      "242.0           1\n",
      "236.0           1\n",
      "231.0           1\n",
      "229.0           1\n",
      "227.0           1\n",
      "224.0           1\n",
      "216.0           1\n",
      "215.0           1\n",
      "212.0           1\n",
      "211.0           1\n",
      "209.0           1\n",
      "207.0           1\n",
      "203.0           1\n",
      "202.0           1\n",
      "200.0           1\n",
      "197.0           1\n",
      "191.0           1\n",
      "187.0           1\n",
      "185.0           1\n",
      "180.0           1\n",
      "179.0           1\n",
      "176.0           1\n",
      "175.0           1\n",
      "174.0           1\n",
      "173.0           1\n",
      "171.0           1\n",
      "162.0           1\n",
      "159.0           1\n",
      "157.0           1\n",
      "150.0           1\n",
      "239.0           1\n",
      "243.0           1\n",
      "332.0           1\n",
      "244.0           1\n",
      "327.0           1\n",
      "325.0           1\n",
      "324.0           1\n",
      "323.0           1\n",
      "319.0           1\n",
      "315.0           1\n",
      "313.0           1\n",
      "312.0           1\n",
      "310.0           1\n",
      "309.0           1\n",
      "307.0           1\n",
      "305.0           1\n",
      "304.0           1\n",
      "303.0           1\n",
      "287.0           1\n",
      "285.0           1\n",
      "283.0           1\n",
      "282.0           1\n",
      "280.0           1\n",
      "278.0           1\n",
      "273.0           1\n",
      "270.0           1\n",
      "269.0           1\n",
      "262.0           1\n",
      "256.0           1\n",
      "253.0           1\n",
      "250.0           1\n",
      "247.0           1\n",
      "246.0           1\n",
      "1722763.0       1\n",
      "Name: count, dtype: int64\n",
      "sodium    \n",
      "2.0           192\n",
      "3.0           167\n",
      "1.0           158\n",
      "4.0           154\n",
      "5.0           118\n",
      "             ... \n",
      "1141.0          1\n",
      "1139.0          1\n",
      "1134.0          1\n",
      "2174.0          1\n",
      "27675110.0      1\n",
      "Name: count, Length: 2434, dtype: int64\n",
      "cakeweek\n",
      "0.0         20042\n",
      "1.0             6\n",
      "24.0            1\n",
      "880.0           1\n",
      "1188.0          1\n",
      "Name: count, dtype: int64\n",
      "wasteless\n",
      "0.0          20049\n",
      "1.0              1\n",
      "1439.0           1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "for col in not_binary_cols:\n",
    "    print(temp_df[[col]].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "afee41fc-b11e-481a-8529-944c9414a33b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- title: string (nullable = true)\n",
      " |-- rating: string (nullable = true)\n",
      " |-- calories: string (nullable = true)\n",
      " |-- protein: double (nullable = true)\n",
      " |-- fat: double (nullable = true)\n",
      " |-- sodium: double (nullable = true)\n",
      " |-- cakeweek: double (nullable = true)\n",
      " |-- wasteless: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.select(not_binary_cols).printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4f111cd1-65ce-4ffe-b67f-d5c48b66ad88",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------------+------------------+\n",
      "|cakeweek                       |wasteless         |\n",
      "+-------------------------------+------------------+\n",
      "|[0.0, 1.0, 1188.0, 24.0, 880.0]|[0.0, 1.0, 1439.0]|\n",
      "+-------------------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.agg(*[F.collect_set(x).alias(x) for x in (\"cakeweek\",\"wasteless\")]).show(truncate=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36c29638-c22f-49e2-ac4c-5212f036116f",
   "metadata": {},
   "source": [
    "其实查看`cakeweek`和`wasteless`值的频率分布，基本数值都是0，要查看和标签的相关关系，再做处理，书上是直接删掉非0和1的记录，这是要结合title中的数值跑到rating列中了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "101d43ce-ff59-4d29-bd66-f5829df705c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+-------------+-------+-----+------+--------+---------+\n",
      "|               title|              rating|     calories|protein|  fat|sodium|cakeweek|wasteless|\n",
      "+--------------------+--------------------+-------------+-------+-----+------+--------+---------+\n",
      "|\"Beet Ravioli wit...| Aged Balsamic Vi...|        4.375|  266.0|  6.0|  21.0|   880.0|      0.0|\n",
      "|\"Seafood \"\"Catapl...|            Vermouth| and Sorrel \"|  4.375|563.0|  49.0|    24.0|   1439.0|\n",
      "|\"\"\"Pot Roast\"\" of...| Aunt Gloria-Style \"|          2.5|  528.0| 38.0|   7.0|  1188.0|      0.0|\n",
      "+--------------------+--------------------+-------------+-------+-----+------+--------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.where(\"cakeweek>1.0 or wasteless>1.0\").select(not_binary_cols).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7d4c2bbc-f878-48f2-9f6e-6c2d4c0ae180",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20054 680\n"
     ]
    }
   ],
   "source": [
    "food = food.where(F.col(\"cakeweek\").isin([0.0,1.0])|F.col(\"cakeweek\").isNull()\n",
    "                 & F.col(\"wasteless\").isin([0.0,1.0])|F.col(\"wasteless\").isNull())\n",
    "print(food.count(),len(food.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "814b249a-da94-4e1f-9600-d08dfcd467ac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- title: string (nullable = true)\n",
      " |-- rating: string (nullable = true)\n",
      " |-- calories: string (nullable = true)\n",
      " |-- protein: double (nullable = true)\n",
      " |-- fat: double (nullable = true)\n",
      " |-- sodium: double (nullable = true)\n",
      " |-- cakeweek: double (nullable = true)\n",
      " |-- wasteless: double (nullable = true)\n",
      " |-- 22_minute_meals: double (nullable = true)\n",
      " |-- 3_ingredient_recipes: double (nullable = true)\n",
      " |-- 30_days_of_groceries: double (nullable = true)\n",
      " |-- advance_prep_required: double (nullable = true)\n",
      " |-- alabama: double (nullable = true)\n",
      " |-- alaska: double (nullable = true)\n",
      " |-- alcoholic: double (nullable = true)\n",
      " |-- almond: double (nullable = true)\n",
      " |-- amaretto: double (nullable = true)\n",
      " |-- anchovy: double (nullable = true)\n",
      " |-- anise: double (nullable = true)\n",
      " |-- anniversary: double (nullable = true)\n",
      " |-- anthony_bourdain: double (nullable = true)\n",
      " |-- aperitif: double (nullable = true)\n",
      " |-- appetizer: double (nullable = true)\n",
      " |-- apple: double (nullable = true)\n",
      " |-- apple_juice: double (nullable = true)\n",
      " |-- apricot: double (nullable = true)\n",
      " |-- arizona: double (nullable = true)\n",
      " |-- artichoke: double (nullable = true)\n",
      " |-- arugula: double (nullable = true)\n",
      " |-- asian_pear: double (nullable = true)\n",
      " |-- asparagus: double (nullable = true)\n",
      " |-- aspen: double (nullable = true)\n",
      " |-- atlanta: double (nullable = true)\n",
      " |-- australia: double (nullable = true)\n",
      " |-- avocado: double (nullable = true)\n",
      " |-- back_to_school: double (nullable = true)\n",
      " |-- backyard_bbq: double (nullable = true)\n",
      " |-- bacon: double (nullable = true)\n",
      " |-- bake: double (nullable = true)\n",
      " |-- banana: double (nullable = true)\n",
      " |-- barley: double (nullable = true)\n",
      " |-- basil: double (nullable = true)\n",
      " |-- bass: double (nullable = true)\n",
      " |-- bastille_day: double (nullable = true)\n",
      " |-- bean: double (nullable = true)\n",
      " |-- beef: double (nullable = true)\n",
      " |-- beef_rib: double (nullable = true)\n",
      " |-- beef_shank: double (nullable = true)\n",
      " |-- beef_tenderloin: double (nullable = true)\n",
      " |-- beer: double (nullable = true)\n",
      " |-- beet: double (nullable = true)\n",
      " |-- bell_pepper: double (nullable = true)\n",
      " |-- berry: double (nullable = true)\n",
      " |-- beverly_hills: double (nullable = true)\n",
      " |-- birthday: double (nullable = true)\n",
      " |-- biscuit: double (nullable = true)\n",
      " |-- bitters: double (nullable = true)\n",
      " |-- blackberry: double (nullable = true)\n",
      " |-- blender: double (nullable = true)\n",
      " |-- blue_cheese: double (nullable = true)\n",
      " |-- blueberry: double (nullable = true)\n",
      " |-- boil: double (nullable = true)\n",
      " |-- bok_choy: double (nullable = true)\n",
      " |-- bon_appétit: double (nullable = true)\n",
      " |-- bon_apptit: double (nullable = true)\n",
      " |-- boston: double (nullable = true)\n",
      " |-- bourbon: double (nullable = true)\n",
      " |-- braise: double (nullable = true)\n",
      " |-- bran: double (nullable = true)\n",
      " |-- brandy: double (nullable = true)\n",
      " |-- bread: double (nullable = true)\n",
      " |-- breadcrumbs: double (nullable = true)\n",
      " |-- breakfast: double (nullable = true)\n",
      " |-- brie: double (nullable = true)\n",
      " |-- brine: double (nullable = true)\n",
      " |-- brisket: double (nullable = true)\n",
      " |-- broccoli: double (nullable = true)\n",
      " |-- broccoli_rabe: double (nullable = true)\n",
      " |-- broil: double (nullable = true)\n",
      " |-- brooklyn: double (nullable = true)\n",
      " |-- brown_rice: double (nullable = true)\n",
      " |-- brownie: double (nullable = true)\n",
      " |-- brunch: double (nullable = true)\n",
      " |-- brussel_sprout: double (nullable = true)\n",
      " |-- buffalo: double (nullable = true)\n",
      " |-- buffet: double (nullable = true)\n",
      " |-- bulgaria: double (nullable = true)\n",
      " |-- bulgur: double (nullable = true)\n",
      " |-- burrito: double (nullable = true)\n",
      " |-- butter: double (nullable = true)\n",
      " |-- buttermilk: double (nullable = true)\n",
      " |-- butternut_squash: double (nullable = true)\n",
      " |-- butterscotch_caramel: double (nullable = true)\n",
      " |-- cabbage: double (nullable = true)\n",
      " |-- cake: double (nullable = true)\n",
      " |-- california: double (nullable = true)\n",
      " |-- calvados: double (nullable = true)\n",
      " |-- cambridge: double (nullable = true)\n",
      " |-- campari: double (nullable = true)\n",
      " |-- camping: double (nullable = true)\n",
      " |-- canada: double (nullable = true)\n",
      " |-- candy: double (nullable = true)\n",
      " |-- candy_thermometer: double (nullable = true)\n",
      " |-- cantaloupe: double (nullable = true)\n",
      " |-- capers: double (nullable = true)\n",
      " |-- caraway: double (nullable = true)\n",
      " |-- cardamom: double (nullable = true)\n",
      " |-- carrot: double (nullable = true)\n",
      " |-- cashew: double (nullable = true)\n",
      " |-- casserole_gratin: double (nullable = true)\n",
      " |-- cauliflower: double (nullable = true)\n",
      " |-- caviar: double (nullable = true)\n",
      " |-- celery: double (nullable = true)\n",
      " |-- chambord: double (nullable = true)\n",
      " |-- champagne: double (nullable = true)\n",
      " |-- chard: double (nullable = true)\n",
      " |-- chartreuse: double (nullable = true)\n",
      " |-- cheddar: double (nullable = true)\n",
      " |-- cheese: double (nullable = true)\n",
      " |-- cherry: double (nullable = true)\n",
      " |-- chestnut: double (nullable = true)\n",
      " |-- chicago: double (nullable = true)\n",
      " |-- chicken: double (nullable = true)\n",
      " |-- chickpea: double (nullable = true)\n",
      " |-- chile: double (nullable = true)\n",
      " |-- chile_pepper: double (nullable = true)\n",
      " |-- chili: double (nullable = true)\n",
      " |-- chill: double (nullable = true)\n",
      " |-- chive: double (nullable = true)\n",
      " |-- chocolate: double (nullable = true)\n",
      " |-- christmas: double (nullable = true)\n",
      " |-- christmas_eve: double (nullable = true)\n",
      " |-- cilantro: double (nullable = true)\n",
      " |-- cinco_de_mayo: double (nullable = true)\n",
      " |-- cinnamon: double (nullable = true)\n",
      " |-- citrus: double (nullable = true)\n",
      " |-- clam: double (nullable = true)\n",
      " |-- clove: double (nullable = true)\n",
      " |-- cobbler_crumble: double (nullable = true)\n",
      " |-- cocktail: double (nullable = true)\n",
      " |-- cocktail_party: double (nullable = true)\n",
      " |-- coconut: double (nullable = true)\n",
      " |-- cod: double (nullable = true)\n",
      " |-- coffee: double (nullable = true)\n",
      " |-- coffee_grinder: double (nullable = true)\n",
      " |-- cognac_armagnac: double (nullable = true)\n",
      " |-- collard_greens: double (nullable = true)\n",
      " |-- colorado: double (nullable = true)\n",
      " |-- columbus: double (nullable = true)\n",
      " |-- condiment: double (nullable = true)\n",
      " |-- condiment_spread: double (nullable = true)\n",
      " |-- connecticut: double (nullable = true)\n",
      " |-- cook_like_a_diner: double (nullable = true)\n",
      " |-- cookbook_critic: double (nullable = true)\n",
      " |-- cookie: double (nullable = true)\n",
      " |-- cookies: double (nullable = true)\n",
      " |-- coriander: double (nullable = true)\n",
      " |-- corn: double (nullable = true)\n",
      " |-- cornmeal: double (nullable = true)\n",
      " |-- costa_mesa: double (nullable = true)\n",
      " |-- cottage_cheese: double (nullable = true)\n",
      " |-- couscous: double (nullable = true)\n",
      " |-- crab: double (nullable = true)\n",
      " |-- cranberry: double (nullable = true)\n",
      " |-- cranberry_sauce: double (nullable = true)\n",
      " |-- cream_cheese: double (nullable = true)\n",
      " |-- créme_de_cacao: double (nullable = true)\n",
      " |-- crêpe: double (nullable = true)\n",
      " |-- crme_de_cacao: double (nullable = true)\n",
      " |-- cuba: double (nullable = true)\n",
      " |-- cucumber: double (nullable = true)\n",
      " |-- cumin: double (nullable = true)\n",
      " |-- cupcake: double (nullable = true)\n",
      " |-- currant: double (nullable = true)\n",
      " |-- curry: double (nullable = true)\n",
      " |-- custard: double (nullable = true)\n",
      " |-- dairy: double (nullable = true)\n",
      " |-- dairy_free: double (nullable = true)\n",
      " |-- dallas: double (nullable = true)\n",
      " |-- date: double (nullable = true)\n",
      " |-- deep_fry: double (nullable = true)\n",
      " |-- denver: double (nullable = true)\n",
      " |-- dessert: double (nullable = true)\n",
      " |-- digestif: double (nullable = true)\n",
      " |-- dill: double (nullable = true)\n",
      " |-- dinner: double (nullable = true)\n",
      " |-- dip: double (nullable = true)\n",
      " |-- diwali: double (nullable = true)\n",
      " |-- dominican_republic: double (nullable = true)\n",
      " |-- dorie_greenspan: double (nullable = true)\n",
      " |-- double_boiler: double (nullable = true)\n",
      " |-- dried_fruit: double (nullable = true)\n",
      " |-- drink: double (nullable = true)\n",
      " |-- drinks: double (nullable = true)\n",
      " |-- duck: double (nullable = true)\n",
      " |-- easter: double (nullable = true)\n",
      " |-- eau_de_vie: double (nullable = true)\n",
      " |-- edible_gift: double (nullable = true)\n",
      " |-- egg: double (nullable = true)\n",
      " |-- egg_nog: double (nullable = true)\n",
      " |-- eggplant: double (nullable = true)\n",
      " |-- egypt: double (nullable = true)\n",
      " |-- emeril_lagasse: double (nullable = true)\n",
      " |-- endive: double (nullable = true)\n",
      " |-- engagement_party: double (nullable = true)\n",
      " |-- england: double (nullable = true)\n",
      " |-- entertaining: double (nullable = true)\n",
      " |-- epi__ushg: double (nullable = true)\n",
      " |-- epi_loves_the_microwave: double (nullable = true)\n",
      " |-- escarole: double (nullable = true)\n",
      " |-- fall: double (nullable = true)\n",
      " |-- family_reunion: double (nullable = true)\n",
      " |-- fat_free: double (nullable = true)\n",
      " |-- fathers_day: double (nullable = true)\n",
      " |-- fennel: double (nullable = true)\n",
      " |-- feta: double (nullable = true)\n",
      " |-- fig: double (nullable = true)\n",
      " |-- fish: double (nullable = true)\n",
      " |-- flaming_hot_summer: double (nullable = true)\n",
      " |-- flat_bread: double (nullable = true)\n",
      " |-- florida: double (nullable = true)\n",
      " |-- fontina: double (nullable = true)\n",
      " |-- food_processor: double (nullable = true)\n",
      " |-- fortified_wine: double (nullable = true)\n",
      " |-- fourth_of_july: double (nullable = true)\n",
      " |-- france: double (nullable = true)\n",
      " |-- frangelico: double (nullable = true)\n",
      " |-- frankenrecipe: double (nullable = true)\n",
      " |-- freeze_chill: double (nullable = true)\n",
      " |-- freezer_food: double (nullable = true)\n",
      " |-- friendsgiving: double (nullable = true)\n",
      " |-- frittata: double (nullable = true)\n",
      " |-- fritter: double (nullable = true)\n",
      " |-- frozen_dessert: double (nullable = true)\n",
      " |-- fruit: double (nullable = true)\n",
      " |-- fruit_juice: double (nullable = true)\n",
      " |-- fry: double (nullable = true)\n",
      " |-- game: double (nullable = true)\n",
      " |-- garlic: double (nullable = true)\n",
      " |-- georgia: double (nullable = true)\n",
      " |-- germany: double (nullable = true)\n",
      " |-- gin: double (nullable = true)\n",
      " |-- ginger: double (nullable = true)\n",
      " |-- goat_cheese: double (nullable = true)\n",
      " |-- goose: double (nullable = true)\n",
      " |-- gouda: double (nullable = true)\n",
      " |-- gourmet: double (nullable = true)\n",
      " |-- graduation: double (nullable = true)\n",
      " |-- grains: double (nullable = true)\n",
      " |-- grand_marnier: double (nullable = true)\n",
      " |-- granola: double (nullable = true)\n",
      " |-- grape: double (nullable = true)\n",
      " |-- grapefruit: double (nullable = true)\n",
      " |-- grappa: double (nullable = true)\n",
      " |-- green_bean: double (nullable = true)\n",
      " |-- green_onion_scallion: double (nullable = true)\n",
      " |-- grill: double (nullable = true)\n",
      " |-- grill_barbecue: double (nullable = true)\n",
      " |-- ground_beef: double (nullable = true)\n",
      " |-- ground_lamb: double (nullable = true)\n",
      " |-- guam: double (nullable = true)\n",
      " |-- guava: double (nullable = true)\n",
      " |-- haiti: double (nullable = true)\n",
      " |-- halibut: double (nullable = true)\n",
      " |-- halloween: double (nullable = true)\n",
      " |-- ham: double (nullable = true)\n",
      " |-- hamburger: double (nullable = true)\n",
      " |-- hanukkah: double (nullable = true)\n",
      " |-- harpercollins: double (nullable = true)\n",
      " |-- hawaii: double (nullable = true)\n",
      " |-- hazelnut: double (nullable = true)\n",
      " |-- healdsburg: double (nullable = true)\n",
      " |-- healthy: double (nullable = true)\n",
      " |-- herb: double (nullable = true)\n",
      " |-- high_fiber: double (nullable = true)\n",
      " |-- hollywood: double (nullable = true)\n",
      " |-- hominy_cornmeal_masa: double (nullable = true)\n",
      " |-- honey: double (nullable = true)\n",
      " |-- honeydew: double (nullable = true)\n",
      " |-- hors_doeuvre: double (nullable = true)\n",
      " |-- horseradish: double (nullable = true)\n",
      " |-- hot_drink: double (nullable = true)\n",
      " |-- hot_pepper: double (nullable = true)\n",
      " |-- house_and_garden: double (nullable = true)\n",
      " |-- house_cocktail: double (nullable = true)\n",
      " |-- houston: double (nullable = true)\n",
      " |-- hummus: double (nullable = true)\n",
      " |-- ice_cream: double (nullable = true)\n",
      " |-- ice_cream_machine: double (nullable = true)\n",
      " |-- iced_coffee: double (nullable = true)\n",
      " |-- iced_tea: double (nullable = true)\n",
      " |-- idaho: double (nullable = true)\n",
      " |-- illinois: double (nullable = true)\n",
      " |-- indiana: double (nullable = true)\n",
      " |-- iowa: double (nullable = true)\n",
      " |-- ireland: double (nullable = true)\n",
      " |-- israel: double (nullable = true)\n",
      " |-- italy: double (nullable = true)\n",
      " |-- jalapeño: double (nullable = true)\n",
      " |-- jam_or_jelly: double (nullable = true)\n",
      " |-- jamaica: double (nullable = true)\n",
      " |-- japan: double (nullable = true)\n",
      " |-- jerusalem_artichoke: double (nullable = true)\n",
      " |-- juicer: double (nullable = true)\n",
      " |-- jícama: double (nullable = true)\n",
      " |-- kahlúa: double (nullable = true)\n",
      " |-- kale: double (nullable = true)\n",
      " |-- kansas: double (nullable = true)\n",
      " |-- kansas_city: double (nullable = true)\n",
      " |-- kentucky: double (nullable = true)\n",
      " |-- kentucky_derby: double (nullable = true)\n",
      " |-- kid_friendly: double (nullable = true)\n",
      " |-- kidney_friendly: double (nullable = true)\n",
      " |-- kirsch: double (nullable = true)\n",
      " |-- kitchen_olympics: double (nullable = true)\n",
      " |-- kiwi: double (nullable = true)\n",
      " |-- kosher: double (nullable = true)\n",
      " |-- kosher_for_passover: double (nullable = true)\n",
      " |-- kumquat: double (nullable = true)\n",
      " |-- kwanzaa: double (nullable = true)\n",
      " |-- labor_day: double (nullable = true)\n",
      " |-- lamb: double (nullable = true)\n",
      " |-- lamb_chop: double (nullable = true)\n",
      " |-- lamb_shank: double (nullable = true)\n",
      " |-- lancaster: double (nullable = true)\n",
      " |-- las_vegas: double (nullable = true)\n",
      " |-- lasagna: double (nullable = true)\n",
      " |-- leafy_green: double (nullable = true)\n",
      " |-- leek: double (nullable = true)\n",
      " |-- legume: double (nullable = true)\n",
      " |-- lemon: double (nullable = true)\n",
      " |-- lemon_juice: double (nullable = true)\n",
      " |-- lemongrass: double (nullable = true)\n",
      " |-- lentil: double (nullable = true)\n",
      " |-- lettuce: double (nullable = true)\n",
      " |-- lima_bean: double (nullable = true)\n",
      " |-- lime: double (nullable = true)\n",
      " |-- lime_juice: double (nullable = true)\n",
      " |-- lingonberry: double (nullable = true)\n",
      " |-- liqueur: double (nullable = true)\n",
      " |-- lobster: double (nullable = true)\n",
      " |-- london: double (nullable = true)\n",
      " |-- long_beach: double (nullable = true)\n",
      " |-- los_angeles: double (nullable = true)\n",
      " |-- louisiana: double (nullable = true)\n",
      " |-- louisville: double (nullable = true)\n",
      " |-- low_cal: double (nullable = true)\n",
      " |-- low_carb: double (nullable = true)\n",
      " |-- low_cholesterol: double (nullable = true)\n",
      " |-- low_fat: double (nullable = true)\n",
      " |-- low_sodium: double (nullable = true)\n",
      " |-- low_sugar: double (nullable = true)\n",
      " |-- low_no_sugar: double (nullable = true)\n",
      " |-- lunar_new_year: double (nullable = true)\n",
      " |-- lunch: double (nullable = true)\n",
      " |-- lychee: double (nullable = true)\n",
      " |-- macadamia_nut: double (nullable = true)\n",
      " |-- macaroni_and_cheese: double (nullable = true)\n",
      " |-- maine: double (nullable = true)\n",
      " |-- mandoline: double (nullable = true)\n",
      " |-- mango: double (nullable = true)\n",
      " |-- maple_syrup: double (nullable = true)\n",
      " |-- mardi_gras: double (nullable = true)\n",
      " |-- margarita: double (nullable = true)\n",
      " |-- marinade: double (nullable = true)\n",
      " |-- marinate: double (nullable = true)\n",
      " |-- marsala: double (nullable = true)\n",
      " |-- marscarpone: double (nullable = true)\n",
      " |-- marshmallow: double (nullable = true)\n",
      " |-- martini: double (nullable = true)\n",
      " |-- maryland: double (nullable = true)\n",
      " |-- massachusetts: double (nullable = true)\n",
      " |-- mayonnaise: double (nullable = true)\n",
      " |-- meat: double (nullable = true)\n",
      " |-- meatball: double (nullable = true)\n",
      " |-- meatloaf: double (nullable = true)\n",
      " |-- melon: double (nullable = true)\n",
      " |-- mexico: double (nullable = true)\n",
      " |-- mezcal: double (nullable = true)\n",
      " |-- miami: double (nullable = true)\n",
      " |-- michigan: double (nullable = true)\n",
      " |-- microwave: double (nullable = true)\n",
      " |-- midori: double (nullable = true)\n",
      " |-- milk_cream: double (nullable = true)\n",
      " |-- minneapolis: double (nullable = true)\n",
      " |-- minnesota: double (nullable = true)\n",
      " |-- mint: double (nullable = true)\n",
      " |-- mississippi: double (nullable = true)\n",
      " |-- missouri: double (nullable = true)\n",
      " |-- mixer: double (nullable = true)\n",
      " |-- molasses: double (nullable = true)\n",
      " |-- monterey_jack: double (nullable = true)\n",
      " |-- mortar_and_pestle: double (nullable = true)\n",
      " |-- mothers_day: double (nullable = true)\n",
      " |-- mozzarella: double (nullable = true)\n",
      " |-- muffin: double (nullable = true)\n",
      " |-- mushroom: double (nullable = true)\n",
      " |-- mussel: double (nullable = true)\n",
      " |-- mustard: double (nullable = true)\n",
      " |-- mustard_greens: double (nullable = true)\n",
      " |-- nancy_silverton: double (nullable = true)\n",
      " |-- nebraska: double (nullable = true)\n",
      " |-- nectarine: double (nullable = true)\n",
      " |-- new_hampshire: double (nullable = true)\n",
      " |-- new_jersey: double (nullable = true)\n",
      " |-- new_mexico: double (nullable = true)\n",
      " |-- new_orleans: double (nullable = true)\n",
      " |-- new_years_day: double (nullable = true)\n",
      " |-- new_years_eve: double (nullable = true)\n",
      " |-- new_york: double (nullable = true)\n",
      " |-- no_meat_no_problem: double (nullable = true)\n",
      " |-- no_sugar_added: double (nullable = true)\n",
      " |-- no_cook: double (nullable = true)\n",
      " |-- non_alcoholic: double (nullable = true)\n",
      " |-- noodle: double (nullable = true)\n",
      " |-- north_carolina: double (nullable = true)\n",
      " |-- nut: double (nullable = true)\n",
      " |-- nutmeg: double (nullable = true)\n",
      " |-- oat: double (nullable = true)\n",
      " |-- oatmeal: double (nullable = true)\n",
      " |-- octopus: double (nullable = true)\n",
      " |-- ohio: double (nullable = true)\n",
      " |-- oklahoma: double (nullable = true)\n",
      " |-- okra: double (nullable = true)\n",
      " |-- oktoberfest: double (nullable = true)\n",
      " |-- olive: double (nullable = true)\n",
      " |-- omelet: double (nullable = true)\n",
      " |-- one_pot_meal: double (nullable = true)\n",
      " |-- onion: double (nullable = true)\n",
      " |-- orange: double (nullable = true)\n",
      " |-- orange_juice: double (nullable = true)\n",
      " |-- oregano: double (nullable = true)\n",
      " |-- oregon: double (nullable = true)\n",
      " |-- organic: double (nullable = true)\n",
      " |-- orzo: double (nullable = true)\n",
      " |-- oscars: double (nullable = true)\n",
      " |-- oyster: double (nullable = true)\n",
      " |-- pacific_palisades: double (nullable = true)\n",
      " |-- paleo: double (nullable = true)\n",
      " |-- pan_fry: double (nullable = true)\n",
      " |-- pancake: double (nullable = true)\n",
      " |-- papaya: double (nullable = true)\n",
      " |-- paprika: double (nullable = true)\n",
      " |-- parade: double (nullable = true)\n",
      " |-- paris: double (nullable = true)\n",
      " |-- parmesan: double (nullable = true)\n",
      " |-- parsley: double (nullable = true)\n",
      " |-- parsnip: double (nullable = true)\n",
      " |-- party: double (nullable = true)\n",
      " |-- pasadena: double (nullable = true)\n",
      " |-- passion_fruit: double (nullable = true)\n",
      " |-- passover: double (nullable = true)\n",
      " |-- pasta: double (nullable = true)\n",
      " |-- pasta_maker: double (nullable = true)\n",
      " |-- pastry: double (nullable = true)\n",
      " |-- pea: double (nullable = true)\n",
      " |-- peach: double (nullable = true)\n",
      " |-- peanut: double (nullable = true)\n",
      " |-- peanut_butter: double (nullable = true)\n",
      " |-- peanut_free: double (nullable = true)\n",
      " |-- pear: double (nullable = true)\n",
      " |-- pecan: double (nullable = true)\n",
      " |-- pennsylvania: double (nullable = true)\n",
      " |-- pepper: double (nullable = true)\n",
      " |-- pernod: double (nullable = true)\n",
      " |-- persian_new_year: double (nullable = true)\n",
      " |-- persimmon: double (nullable = true)\n",
      " |-- peru: double (nullable = true)\n",
      " |-- pescatarian: double (nullable = true)\n",
      " |-- philippines: double (nullable = true)\n",
      " |-- phyllo_puff_pastry_dough: double (nullable = true)\n",
      " |-- pickles: double (nullable = true)\n",
      " |-- picnic: double (nullable = true)\n",
      " |-- pie: double (nullable = true)\n",
      " |-- pine_nut: double (nullable = true)\n",
      " |-- pineapple: double (nullable = true)\n",
      " |-- pistachio: double (nullable = true)\n",
      " |-- pittsburgh: double (nullable = true)\n",
      " |-- pizza: double (nullable = true)\n",
      " |-- plantain: double (nullable = true)\n",
      " |-- plum: double (nullable = true)\n",
      " |-- poach: double (nullable = true)\n",
      " |-- poblano: double (nullable = true)\n",
      " |-- poker_game_night: double (nullable = true)\n",
      " |-- pomegranate: double (nullable = true)\n",
      " |-- pomegranate_juice: double (nullable = true)\n",
      " |-- poppy: double (nullable = true)\n",
      " |-- pork: double (nullable = true)\n",
      " |-- pork_chop: double (nullable = true)\n",
      " |-- pork_rib: double (nullable = true)\n",
      " |-- pork_tenderloin: double (nullable = true)\n",
      " |-- port: double (nullable = true)\n",
      " |-- portland: double (nullable = true)\n",
      " |-- pot_pie: double (nullable = true)\n",
      " |-- potato: double (nullable = true)\n",
      " |-- potato_salad: double (nullable = true)\n",
      " |-- potluck: double (nullable = true)\n",
      " |-- poultry: double (nullable = true)\n",
      " |-- poultry_sausage: double (nullable = true)\n",
      " |-- pressure_cooker: double (nullable = true)\n",
      " |-- prosciutto: double (nullable = true)\n",
      " |-- providence: double (nullable = true)\n",
      " |-- prune: double (nullable = true)\n",
      " |-- pumpkin: double (nullable = true)\n",
      " |-- punch: double (nullable = true)\n",
      " |-- purim: double (nullable = true)\n",
      " |-- quail: double (nullable = true)\n",
      " |-- quiche: double (nullable = true)\n",
      " |-- quick_and_easy: double (nullable = true)\n",
      " |-- quick_and_healthy: double (nullable = true)\n",
      " |-- quince: double (nullable = true)\n",
      " |-- quinoa: double (nullable = true)\n",
      " |-- rabbit: double (nullable = true)\n",
      " |-- rack_of_lamb: double (nullable = true)\n",
      " |-- radicchio: double (nullable = true)\n",
      " |-- radish: double (nullable = true)\n",
      " |-- raisin: double (nullable = true)\n",
      " |-- ramadan: double (nullable = true)\n",
      " |-- ramekin: double (nullable = true)\n",
      " |-- raspberry: double (nullable = true)\n",
      " |-- raw: double (nullable = true)\n",
      " |-- red_wine: double (nullable = true)\n",
      " |-- rhode_island: double (nullable = true)\n",
      " |-- rhubarb: double (nullable = true)\n",
      " |-- rice: double (nullable = true)\n",
      " |-- ricotta: double (nullable = true)\n",
      " |-- roast: double (nullable = true)\n",
      " |-- root_vegetable: double (nullable = true)\n",
      " |-- rosemary: double (nullable = true)\n",
      " |-- rosh_hashanah_yom_kippur: double (nullable = true)\n",
      " |-- rosé: double (nullable = true)\n",
      " |-- rub: double (nullable = true)\n",
      " |-- rum: double (nullable = true)\n",
      " |-- rutabaga: double (nullable = true)\n",
      " |-- rye: double (nullable = true)\n",
      " |-- saffron: double (nullable = true)\n",
      " |-- sage: double (nullable = true)\n",
      " |-- sake: double (nullable = true)\n",
      " |-- salad: double (nullable = true)\n",
      " |-- salad_dressing: double (nullable = true)\n",
      " |-- salmon: double (nullable = true)\n",
      " |-- salsa: double (nullable = true)\n",
      " |-- san_francisco: double (nullable = true)\n",
      " |-- sandwich: double (nullable = true)\n",
      " |-- sandwich_theory: double (nullable = true)\n",
      " |-- sangria: double (nullable = true)\n",
      " |-- santa_monica: double (nullable = true)\n",
      " |-- sardine: double (nullable = true)\n",
      " |-- sauce: double (nullable = true)\n",
      " |-- sausage: double (nullable = true)\n",
      " |-- sauté: double (nullable = true)\n",
      " |-- scallop: double (nullable = true)\n",
      " |-- scotch: double (nullable = true)\n",
      " |-- seafood: double (nullable = true)\n",
      " |-- seattle: double (nullable = true)\n",
      " |-- seed: double (nullable = true)\n",
      " |-- self: double (nullable = true)\n",
      " |-- semolina: double (nullable = true)\n",
      " |-- sesame: double (nullable = true)\n",
      " |-- sesame_oil: double (nullable = true)\n",
      " |-- shallot: double (nullable = true)\n",
      " |-- shavuot: double (nullable = true)\n",
      " |-- shellfish: double (nullable = true)\n",
      " |-- sherry: double (nullable = true)\n",
      " |-- shower: double (nullable = true)\n",
      " |-- shrimp: double (nullable = true)\n",
      " |-- side: double (nullable = true)\n",
      " |-- simmer: double (nullable = true)\n",
      " |-- skewer: double (nullable = true)\n",
      " |-- slow_cooker: double (nullable = true)\n",
      " |-- smoker: double (nullable = true)\n",
      " |-- smoothie: double (nullable = true)\n",
      " |-- snapper: double (nullable = true)\n",
      " |-- sorbet: double (nullable = true)\n",
      " |-- soufflé_meringue: double (nullable = true)\n",
      " |-- soup_stew: double (nullable = true)\n",
      " |-- sour_cream: double (nullable = true)\n",
      " |-- sourdough: double (nullable = true)\n",
      " |-- south_carolina: double (nullable = true)\n",
      " |-- soy: double (nullable = true)\n",
      " |-- soy_free: double (nullable = true)\n",
      " |-- soy_sauce: double (nullable = true)\n",
      " |-- spain: double (nullable = true)\n",
      " |-- sparkling_wine: double (nullable = true)\n",
      " |-- spice: double (nullable = true)\n",
      " |-- spinach: double (nullable = true)\n",
      " |-- spirit: double (nullable = true)\n",
      " |-- spring: double (nullable = true)\n",
      " |-- spritzer: double (nullable = true)\n",
      " |-- squash: double (nullable = true)\n",
      " |-- squid: double (nullable = true)\n",
      " |-- st_louis: double (nullable = true)\n",
      " |-- st_patricks_day: double (nullable = true)\n",
      " |-- steak: double (nullable = true)\n",
      " |-- steam: double (nullable = true)\n",
      " |-- stew: double (nullable = true)\n",
      " |-- stir_fry: double (nullable = true)\n",
      " |-- stock: double (nullable = true)\n",
      " |-- strawberry: double (nullable = true)\n",
      " |-- stuffing_dressing: double (nullable = true)\n",
      " |-- sugar_conscious: double (nullable = true)\n",
      " |-- sugar_snap_pea: double (nullable = true)\n",
      " |-- sukkot: double (nullable = true)\n",
      " |-- summer: double (nullable = true)\n",
      " |-- super_bowl: double (nullable = true)\n",
      " |-- suzanne_goin: double (nullable = true)\n",
      " |-- sweet_potato_yam: double (nullable = true)\n",
      " |-- swiss_cheese: double (nullable = true)\n",
      " |-- switzerland: double (nullable = true)\n",
      " |-- swordfish: double (nullable = true)\n",
      " |-- taco: double (nullable = true)\n",
      " |-- tailgating: double (nullable = true)\n",
      " |-- tamarind: double (nullable = true)\n",
      " |-- tangerine: double (nullable = true)\n",
      " |-- tapioca: double (nullable = true)\n",
      " |-- tarragon: double (nullable = true)\n",
      " |-- tart: double (nullable = true)\n",
      " |-- tea: double (nullable = true)\n",
      " |-- tennessee: double (nullable = true)\n",
      " |-- tequila: double (nullable = true)\n",
      " |-- tested_and_improved: double (nullable = true)\n",
      " |-- texas: double (nullable = true)\n",
      " |-- thanksgiving: double (nullable = true)\n",
      " |-- thyme: double (nullable = true)\n",
      " |-- tilapia: double (nullable = true)\n",
      " |-- tofu: double (nullable = true)\n",
      " |-- tomatillo: double (nullable = true)\n",
      " |-- tomato: double (nullable = true)\n",
      " |-- tortillas: double (nullable = true)\n",
      " |-- tree_nut: double (nullable = true)\n",
      " |-- tree_nut_free: double (nullable = true)\n",
      " |-- triple_sec: double (nullable = true)\n",
      " |-- tropical_fruit: double (nullable = true)\n",
      " |-- trout: double (nullable = true)\n",
      " |-- tuna: double (nullable = true)\n",
      " |-- turnip: double (nullable = true)\n",
      " |-- utah: double (nullable = true)\n",
      " |-- valentines_day: double (nullable = true)\n",
      " |-- vanilla: double (nullable = true)\n",
      " |-- veal: double (nullable = true)\n",
      " |-- vegan: double (nullable = true)\n",
      " |-- vegetable: double (nullable = true)\n",
      " |-- vegetarian: double (nullable = true)\n",
      " |-- venison: double (nullable = true)\n",
      " |-- vermont: double (nullable = true)\n",
      " |-- vermouth: double (nullable = true)\n",
      " |-- vinegar: double (nullable = true)\n",
      " |-- virginia: double (nullable = true)\n",
      " |-- vodka: double (nullable = true)\n",
      " |-- waffle: double (nullable = true)\n",
      " |-- walnut: double (nullable = true)\n",
      " |-- wasabi: double (nullable = true)\n",
      " |-- washington: double (nullable = true)\n",
      " |-- washington_dc: double (nullable = true)\n",
      " |-- watercress: double (nullable = true)\n",
      " |-- watermelon: double (nullable = true)\n",
      " |-- wedding: double (nullable = true)\n",
      " |-- weelicious: double (nullable = true)\n",
      " |-- west_virginia: double (nullable = true)\n",
      " |-- westwood: double (nullable = true)\n",
      " |-- wheat_gluten_free: double (nullable = true)\n",
      " |-- whiskey: double (nullable = true)\n",
      " |-- white_wine: double (nullable = true)\n",
      " |-- whole_wheat: double (nullable = true)\n",
      " |-- wild_rice: double (nullable = true)\n",
      " |-- windsor: double (nullable = true)\n",
      " |-- wine: double (nullable = true)\n",
      " |-- winter: double (nullable = true)\n",
      " |-- wisconsin: double (nullable = true)\n",
      " |-- wok: double (nullable = true)\n",
      " |-- yellow_squash: double (nullable = true)\n",
      " |-- yogurt: double (nullable = true)\n",
      " |-- yonkers: double (nullable = true)\n",
      " |-- yuca: double (nullable = true)\n",
      " |-- zucchini: double (nullable = true)\n",
      " |-- cookbooks: double (nullable = true)\n",
      " |-- leftovers: double (nullable = true)\n",
      " |-- snack: double (nullable = true)\n",
      " |-- snack_week: double (nullable = true)\n",
      " |-- turkey: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ab02d500-c073-4e6c-83e1-89157023c76e",
   "metadata": {},
   "outputs": [],
   "source": [
    "IDENTIFIERS = [\"title\"]\n",
    "CONTINUOUS_COLUMNS = [\"rating\",\"calories\",\"protein\",\"fat\",\"sodium\"]\n",
    "TARGET_COLUMN = [\"dessert\"]\n",
    "BINARY_COLUMNS = [x for x in food.columns if x not in IDENTIFIERS and x not in  CONTINUOUS_COLUMNS and x not in TARGET_COLUMN]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ff57b2ca-4c71-45cb-a3cf-80999e34af15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20049 680\n"
     ]
    }
   ],
   "source": [
    "food = food.dropna(how=\"all\").dropna(subset=TARGET_COLUMN)\n",
    "print(food.count(),len(food.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5c3a3b8e-3183-4963-84db-db53f705c060",
   "metadata": {},
   "outputs": [],
   "source": [
    "# food = food.fillna(0.0,subset=BINARY_COLUMNS)\n",
    "# sum([food.where(F.col(col).isNull()).count() for col in BINARY_COLUMNS])\n",
    "#返回0,运行得特别慢"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "41ed2991-116f-4b90-b330-1a535f9d3920",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Optional\n",
    "\n",
    "@F.udf(T.BooleanType())\n",
    "def is_a_number(value:Optional[str])->bool:\n",
    "    if not value:\n",
    "        return True\n",
    "    elif value.replace('.','').isnumeric():\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "74a8ccc1-2e24-4a21-be1e-37c79f0e28f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------------+-------+----+------+\n",
      "|   rating|    calories|protein| fat|sodium|\n",
      "+---------+------------+-------+----+------+\n",
      "| Cucumber| and Lemon \"|   3.75|NULL|  NULL|\n",
      "+---------+------------+-------+----+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.where(~is_a_number(F.col(\"rating\"))|~is_a_number(F.col(\"calories\"))).select(CONTINUOUS_COLUMNS).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "44baff04-659d-4b4f-8d92-5b40bf360a01",
   "metadata": {},
   "outputs": [],
   "source": [
    "food = food.where(\n",
    "    is_a_number(F.col(\"rating\"))&is_a_number(F.col(\"calories\"))\n",
    ").withColumns(\n",
    "    {col:F.col(col).cast(\"double\") for col in [\"rating\",\"calories\"]}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "41ed8ec5-8f78-43bc-a9cb-80f07b860556",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method withColumns in module pyspark.sql.dataframe:\n",
      "\n",
      "withColumns(*colsMap: Dict[str, pyspark.sql.column.Column]) -> 'DataFrame' method of pyspark.sql.dataframe.DataFrame instance\n",
      "    Returns a new :class:`DataFrame` by adding multiple columns or replacing the\n",
      "    existing columns that have the same names.\n",
      "    \n",
      "    The colsMap is a map of column name and column, the column must only refer to attributes\n",
      "    supplied by this Dataset. It is an error to add columns that refer to some other Dataset.\n",
      "    \n",
      "    .. versionadded:: 3.3.0\n",
      "       Added support for multiple columns adding\n",
      "    \n",
      "    .. versionchanged:: 3.4.0\n",
      "        Supports Spark Connect.\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    colsMap : dict\n",
      "        a dict of column name and :class:`Column`. Currently, only a single map is supported.\n",
      "    \n",
      "    Returns\n",
      "    -------\n",
      "    :class:`DataFrame`\n",
      "        DataFrame with new or replaced columns.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df = spark.createDataFrame([(2, \"Alice\"), (5, \"Bob\")], schema=[\"age\", \"name\"])\n",
      "    >>> df.withColumns({'age2': df.age + 2, 'age3': df.age + 3}).show()\n",
      "    +---+-----+----+----+\n",
      "    |age| name|age2|age3|\n",
      "    +---+-----+----+----+\n",
      "    |  2|Alice|   4|   5|\n",
      "    |  5|  Bob|   7|   8|\n",
      "    +---+-----+----+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(food.withColumns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5affdd5b-c9ba-44b9-bce1-b27c5d60cf88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20048 680\n"
     ]
    }
   ],
   "source": [
    "print(food.count(),len(food.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c0f72c9b-c527-440a-b499-0f148de92602",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+------------------+------------------+------------------+------------------+\n",
      "|summary|           rating|          calories|           protein|               fat|            sodium|\n",
      "+-------+-----------------+------------------+------------------+------------------+------------------+\n",
      "|   mean|3.714460295291301|6324.0634571930705|100.17385283565179| 346.9398083953107| 6226.927244193346|\n",
      "| stddev|1.340918766050896| 359079.8369634019|3840.6809971287385|20458.040344124103|333349.56803702697|\n",
      "|    min|              0.0|               0.0|               0.0|               0.0|               0.0|\n",
      "|     1%|              0.0|              18.0|               0.0|               0.0|               1.0|\n",
      "|     5%|              0.0|              62.0|               0.0|               0.0|               5.0|\n",
      "|    50%|            4.375|             331.0|               8.0|              17.0|             294.0|\n",
      "|    95%|              5.0|            1315.0|              75.0|              85.0|            2050.0|\n",
      "|    99%|              5.0|            3184.0|             173.0|             207.0|            5649.0|\n",
      "|    max|              5.0|       3.0111218E7|          236489.0|         1722763.0|        2.767511E7|\n",
      "+-------+-----------------+------------------+------------------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.select(*CONTINUOUS_COLUMNS).summary(\n",
    "    \"mean\",\n",
    "    \"stddev\",\n",
    "    \"min\",\n",
    "    \"1%\",\n",
    "    \"5%\",\n",
    "    \"50%\",\n",
    "    \"95%\",\n",
    "    \"99%\",\n",
    "    \"max\"\n",
    ").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4b3c62bb-4ccd-44a1-9224-e247bbbca292",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function least in module pyspark.sql.functions:\n",
      "\n",
      "least(*cols: 'ColumnOrName') -> pyspark.sql.column.Column\n",
      "    Returns the least value of the list of column names, skipping null values.\n",
      "    This function takes at least 2 parameters. It will return null if all parameters are null.\n",
      "    \n",
      "    .. versionadded:: 1.5.0\n",
      "    \n",
      "    .. versionchanged:: 3.4.0\n",
      "        Supports Spark Connect.\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    cols : :class:`~pyspark.sql.Column` or str\n",
      "        column names or columns to be compared\n",
      "    \n",
      "    Returns\n",
      "    -------\n",
      "    :class:`~pyspark.sql.Column`\n",
      "        least value.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])\n",
      "    >>> df.select(least(df.a, df.b, df.c).alias(\"least\")).collect()\n",
      "    [Row(least=1)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(F.least)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ec37e8b6-12e6-4e38-844d-16335a879579",
   "metadata": {},
   "outputs": [],
   "source": [
    "maximum = {\n",
    "    \"calories\":3184.0,\n",
    "    \"protein\":173.0,\n",
    "    \"fat\":207.0,\n",
    "    \"sodium\":5649.0\n",
    "}\n",
    "for k,v in maximum.items():\n",
    "    food = food.withColumn(k,F.when(F.isnull(F.col(k)),F.col(k)).otherwise(F.least(F.col(k),F.lit(v))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "28ad9534-5263-4098-9956-314055f14a31",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Column<'sum(cakeweek) AS cakeweek'>,\n",
       " Column<'sum(wasteless) AS wasteless'>,\n",
       " Column<'sum(22_minute_meals) AS 22_minute_meals'>,\n",
       " Column<'sum(3_ingredient_recipes) AS 3_ingredient_recipes'>,\n",
       " Column<'sum(30_days_of_groceries) AS 30_days_of_groceries'>,\n",
       " Column<'sum(advance_prep_required) AS advance_prep_required'>,\n",
       " Column<'sum(alabama) AS alabama'>,\n",
       " Column<'sum(alaska) AS alaska'>,\n",
       " Column<'sum(alcoholic) AS alcoholic'>,\n",
       " Column<'sum(almond) AS almond'>,\n",
       " Column<'sum(amaretto) AS amaretto'>,\n",
       " Column<'sum(anchovy) AS anchovy'>,\n",
       " Column<'sum(anise) AS anise'>,\n",
       " Column<'sum(anniversary) AS anniversary'>,\n",
       " Column<'sum(anthony_bourdain) AS anthony_bourdain'>,\n",
       " Column<'sum(aperitif) AS aperitif'>,\n",
       " Column<'sum(appetizer) AS appetizer'>,\n",
       " Column<'sum(apple) AS apple'>,\n",
       " Column<'sum(apple_juice) AS apple_juice'>,\n",
       " Column<'sum(apricot) AS apricot'>,\n",
       " Column<'sum(arizona) AS arizona'>,\n",
       " Column<'sum(artichoke) AS artichoke'>,\n",
       " Column<'sum(arugula) AS arugula'>,\n",
       " Column<'sum(asian_pear) AS asian_pear'>,\n",
       " Column<'sum(asparagus) AS asparagus'>,\n",
       " Column<'sum(aspen) AS aspen'>,\n",
       " Column<'sum(atlanta) AS atlanta'>,\n",
       " Column<'sum(australia) AS australia'>,\n",
       " Column<'sum(avocado) AS avocado'>,\n",
       " Column<'sum(back_to_school) AS back_to_school'>,\n",
       " Column<'sum(backyard_bbq) AS backyard_bbq'>,\n",
       " Column<'sum(bacon) AS bacon'>,\n",
       " Column<'sum(bake) AS bake'>,\n",
       " Column<'sum(banana) AS banana'>,\n",
       " Column<'sum(barley) AS barley'>,\n",
       " Column<'sum(basil) AS basil'>,\n",
       " Column<'sum(bass) AS bass'>,\n",
       " Column<'sum(bastille_day) AS bastille_day'>,\n",
       " Column<'sum(bean) AS bean'>,\n",
       " Column<'sum(beef) AS beef'>,\n",
       " Column<'sum(beef_rib) AS beef_rib'>,\n",
       " Column<'sum(beef_shank) AS beef_shank'>,\n",
       " Column<'sum(beef_tenderloin) AS beef_tenderloin'>,\n",
       " Column<'sum(beer) AS beer'>,\n",
       " Column<'sum(beet) AS beet'>,\n",
       " Column<'sum(bell_pepper) AS bell_pepper'>,\n",
       " Column<'sum(berry) AS berry'>,\n",
       " Column<'sum(beverly_hills) AS beverly_hills'>,\n",
       " Column<'sum(birthday) AS birthday'>,\n",
       " Column<'sum(biscuit) AS biscuit'>,\n",
       " Column<'sum(bitters) AS bitters'>,\n",
       " Column<'sum(blackberry) AS blackberry'>,\n",
       " Column<'sum(blender) AS blender'>,\n",
       " Column<'sum(blue_cheese) AS blue_cheese'>,\n",
       " Column<'sum(blueberry) AS blueberry'>,\n",
       " Column<'sum(boil) AS boil'>,\n",
       " Column<'sum(bok_choy) AS bok_choy'>,\n",
       " Column<'sum(bon_appétit) AS `bon_appétit`'>,\n",
       " Column<'sum(bon_apptit) AS bon_apptit'>,\n",
       " Column<'sum(boston) AS boston'>,\n",
       " Column<'sum(bourbon) AS bourbon'>,\n",
       " Column<'sum(braise) AS braise'>,\n",
       " Column<'sum(bran) AS bran'>,\n",
       " Column<'sum(brandy) AS brandy'>,\n",
       " Column<'sum(bread) AS bread'>,\n",
       " Column<'sum(breadcrumbs) AS breadcrumbs'>,\n",
       " Column<'sum(breakfast) AS breakfast'>,\n",
       " Column<'sum(brie) AS brie'>,\n",
       " Column<'sum(brine) AS brine'>,\n",
       " Column<'sum(brisket) AS brisket'>,\n",
       " Column<'sum(broccoli) AS broccoli'>,\n",
       " Column<'sum(broccoli_rabe) AS broccoli_rabe'>,\n",
       " Column<'sum(broil) AS broil'>,\n",
       " Column<'sum(brooklyn) AS brooklyn'>,\n",
       " Column<'sum(brown_rice) AS brown_rice'>,\n",
       " Column<'sum(brownie) AS brownie'>,\n",
       " Column<'sum(brunch) AS brunch'>,\n",
       " Column<'sum(brussel_sprout) AS brussel_sprout'>,\n",
       " Column<'sum(buffalo) AS buffalo'>,\n",
       " Column<'sum(buffet) AS buffet'>,\n",
       " Column<'sum(bulgaria) AS bulgaria'>,\n",
       " Column<'sum(bulgur) AS bulgur'>,\n",
       " Column<'sum(burrito) AS burrito'>,\n",
       " Column<'sum(butter) AS butter'>,\n",
       " Column<'sum(buttermilk) AS buttermilk'>,\n",
       " Column<'sum(butternut_squash) AS butternut_squash'>,\n",
       " Column<'sum(butterscotch_caramel) AS butterscotch_caramel'>,\n",
       " Column<'sum(cabbage) AS cabbage'>,\n",
       " Column<'sum(cake) AS cake'>,\n",
       " Column<'sum(california) AS california'>,\n",
       " Column<'sum(calvados) AS calvados'>,\n",
       " Column<'sum(cambridge) AS cambridge'>,\n",
       " Column<'sum(campari) AS campari'>,\n",
       " Column<'sum(camping) AS camping'>,\n",
       " Column<'sum(canada) AS canada'>,\n",
       " Column<'sum(candy) AS candy'>,\n",
       " Column<'sum(candy_thermometer) AS candy_thermometer'>,\n",
       " Column<'sum(cantaloupe) AS cantaloupe'>,\n",
       " Column<'sum(capers) AS capers'>,\n",
       " Column<'sum(caraway) AS caraway'>,\n",
       " Column<'sum(cardamom) AS cardamom'>,\n",
       " Column<'sum(carrot) AS carrot'>,\n",
       " Column<'sum(cashew) AS cashew'>,\n",
       " Column<'sum(casserole_gratin) AS casserole_gratin'>,\n",
       " Column<'sum(cauliflower) AS cauliflower'>,\n",
       " Column<'sum(caviar) AS caviar'>,\n",
       " Column<'sum(celery) AS celery'>,\n",
       " Column<'sum(chambord) AS chambord'>,\n",
       " Column<'sum(champagne) AS champagne'>,\n",
       " Column<'sum(chard) AS chard'>,\n",
       " Column<'sum(chartreuse) AS chartreuse'>,\n",
       " Column<'sum(cheddar) AS cheddar'>,\n",
       " Column<'sum(cheese) AS cheese'>,\n",
       " Column<'sum(cherry) AS cherry'>,\n",
       " Column<'sum(chestnut) AS chestnut'>,\n",
       " Column<'sum(chicago) AS chicago'>,\n",
       " Column<'sum(chicken) AS chicken'>,\n",
       " Column<'sum(chickpea) AS chickpea'>,\n",
       " Column<'sum(chile) AS chile'>,\n",
       " Column<'sum(chile_pepper) AS chile_pepper'>,\n",
       " Column<'sum(chili) AS chili'>,\n",
       " Column<'sum(chill) AS chill'>,\n",
       " Column<'sum(chive) AS chive'>,\n",
       " Column<'sum(chocolate) AS chocolate'>,\n",
       " Column<'sum(christmas) AS christmas'>,\n",
       " Column<'sum(christmas_eve) AS christmas_eve'>,\n",
       " Column<'sum(cilantro) AS cilantro'>,\n",
       " Column<'sum(cinco_de_mayo) AS cinco_de_mayo'>,\n",
       " Column<'sum(cinnamon) AS cinnamon'>,\n",
       " Column<'sum(citrus) AS citrus'>,\n",
       " Column<'sum(clam) AS clam'>,\n",
       " Column<'sum(clove) AS clove'>,\n",
       " Column<'sum(cobbler_crumble) AS cobbler_crumble'>,\n",
       " Column<'sum(cocktail) AS cocktail'>,\n",
       " Column<'sum(cocktail_party) AS cocktail_party'>,\n",
       " Column<'sum(coconut) AS coconut'>,\n",
       " Column<'sum(cod) AS cod'>,\n",
       " Column<'sum(coffee) AS coffee'>,\n",
       " Column<'sum(coffee_grinder) AS coffee_grinder'>,\n",
       " Column<'sum(cognac_armagnac) AS cognac_armagnac'>,\n",
       " Column<'sum(collard_greens) AS collard_greens'>,\n",
       " Column<'sum(colorado) AS colorado'>,\n",
       " Column<'sum(columbus) AS columbus'>,\n",
       " Column<'sum(condiment) AS condiment'>,\n",
       " Column<'sum(condiment_spread) AS condiment_spread'>,\n",
       " Column<'sum(connecticut) AS connecticut'>,\n",
       " Column<'sum(cook_like_a_diner) AS cook_like_a_diner'>,\n",
       " Column<'sum(cookbook_critic) AS cookbook_critic'>,\n",
       " Column<'sum(cookie) AS cookie'>,\n",
       " Column<'sum(cookies) AS cookies'>,\n",
       " Column<'sum(coriander) AS coriander'>,\n",
       " Column<'sum(corn) AS corn'>,\n",
       " Column<'sum(cornmeal) AS cornmeal'>,\n",
       " Column<'sum(costa_mesa) AS costa_mesa'>,\n",
       " Column<'sum(cottage_cheese) AS cottage_cheese'>,\n",
       " Column<'sum(couscous) AS couscous'>,\n",
       " Column<'sum(crab) AS crab'>,\n",
       " Column<'sum(cranberry) AS cranberry'>,\n",
       " Column<'sum(cranberry_sauce) AS cranberry_sauce'>,\n",
       " Column<'sum(cream_cheese) AS cream_cheese'>,\n",
       " Column<'sum(créme_de_cacao) AS `créme_de_cacao`'>,\n",
       " Column<'sum(crêpe) AS `crêpe`'>,\n",
       " Column<'sum(crme_de_cacao) AS crme_de_cacao'>,\n",
       " Column<'sum(cuba) AS cuba'>,\n",
       " Column<'sum(cucumber) AS cucumber'>,\n",
       " Column<'sum(cumin) AS cumin'>,\n",
       " Column<'sum(cupcake) AS cupcake'>,\n",
       " Column<'sum(currant) AS currant'>,\n",
       " Column<'sum(curry) AS curry'>,\n",
       " Column<'sum(custard) AS custard'>,\n",
       " Column<'sum(dairy) AS dairy'>,\n",
       " Column<'sum(dairy_free) AS dairy_free'>,\n",
       " Column<'sum(dallas) AS dallas'>,\n",
       " Column<'sum(date) AS date'>,\n",
       " Column<'sum(deep_fry) AS deep_fry'>,\n",
       " Column<'sum(denver) AS denver'>,\n",
       " Column<'sum(digestif) AS digestif'>,\n",
       " Column<'sum(dill) AS dill'>,\n",
       " Column<'sum(dinner) AS dinner'>,\n",
       " Column<'sum(dip) AS dip'>,\n",
       " Column<'sum(diwali) AS diwali'>,\n",
       " Column<'sum(dominican_republic) AS dominican_republic'>,\n",
       " Column<'sum(dorie_greenspan) AS dorie_greenspan'>,\n",
       " Column<'sum(double_boiler) AS double_boiler'>,\n",
       " Column<'sum(dried_fruit) AS dried_fruit'>,\n",
       " Column<'sum(drink) AS drink'>,\n",
       " Column<'sum(drinks) AS drinks'>,\n",
       " Column<'sum(duck) AS duck'>,\n",
       " Column<'sum(easter) AS easter'>,\n",
       " Column<'sum(eau_de_vie) AS eau_de_vie'>,\n",
       " Column<'sum(edible_gift) AS edible_gift'>,\n",
       " Column<'sum(egg) AS egg'>,\n",
       " Column<'sum(egg_nog) AS egg_nog'>,\n",
       " Column<'sum(eggplant) AS eggplant'>,\n",
       " Column<'sum(egypt) AS egypt'>,\n",
       " Column<'sum(emeril_lagasse) AS emeril_lagasse'>,\n",
       " Column<'sum(endive) AS endive'>,\n",
       " Column<'sum(engagement_party) AS engagement_party'>,\n",
       " Column<'sum(england) AS england'>,\n",
       " Column<'sum(entertaining) AS entertaining'>,\n",
       " Column<'sum(epi__ushg) AS epi__ushg'>,\n",
       " Column<'sum(epi_loves_the_microwave) AS epi_loves_the_microwave'>,\n",
       " Column<'sum(escarole) AS escarole'>,\n",
       " Column<'sum(fall) AS fall'>,\n",
       " Column<'sum(family_reunion) AS family_reunion'>,\n",
       " Column<'sum(fat_free) AS fat_free'>,\n",
       " Column<'sum(fathers_day) AS fathers_day'>,\n",
       " Column<'sum(fennel) AS fennel'>,\n",
       " Column<'sum(feta) AS feta'>,\n",
       " Column<'sum(fig) AS fig'>,\n",
       " Column<'sum(fish) AS fish'>,\n",
       " Column<'sum(flaming_hot_summer) AS flaming_hot_summer'>,\n",
       " Column<'sum(flat_bread) AS flat_bread'>,\n",
       " Column<'sum(florida) AS florida'>,\n",
       " Column<'sum(fontina) AS fontina'>,\n",
       " Column<'sum(food_processor) AS food_processor'>,\n",
       " Column<'sum(fortified_wine) AS fortified_wine'>,\n",
       " Column<'sum(fourth_of_july) AS fourth_of_july'>,\n",
       " Column<'sum(france) AS france'>,\n",
       " Column<'sum(frangelico) AS frangelico'>,\n",
       " Column<'sum(frankenrecipe) AS frankenrecipe'>,\n",
       " Column<'sum(freeze_chill) AS freeze_chill'>,\n",
       " Column<'sum(freezer_food) AS freezer_food'>,\n",
       " Column<'sum(friendsgiving) AS friendsgiving'>,\n",
       " Column<'sum(frittata) AS frittata'>,\n",
       " Column<'sum(fritter) AS fritter'>,\n",
       " Column<'sum(frozen_dessert) AS frozen_dessert'>,\n",
       " Column<'sum(fruit) AS fruit'>,\n",
       " Column<'sum(fruit_juice) AS fruit_juice'>,\n",
       " Column<'sum(fry) AS fry'>,\n",
       " Column<'sum(game) AS game'>,\n",
       " Column<'sum(garlic) AS garlic'>,\n",
       " Column<'sum(georgia) AS georgia'>,\n",
       " Column<'sum(germany) AS germany'>,\n",
       " Column<'sum(gin) AS gin'>,\n",
       " Column<'sum(ginger) AS ginger'>,\n",
       " Column<'sum(goat_cheese) AS goat_cheese'>,\n",
       " Column<'sum(goose) AS goose'>,\n",
       " Column<'sum(gouda) AS gouda'>,\n",
       " Column<'sum(gourmet) AS gourmet'>,\n",
       " Column<'sum(graduation) AS graduation'>,\n",
       " Column<'sum(grains) AS grains'>,\n",
       " Column<'sum(grand_marnier) AS grand_marnier'>,\n",
       " Column<'sum(granola) AS granola'>,\n",
       " Column<'sum(grape) AS grape'>,\n",
       " Column<'sum(grapefruit) AS grapefruit'>,\n",
       " Column<'sum(grappa) AS grappa'>,\n",
       " Column<'sum(green_bean) AS green_bean'>,\n",
       " Column<'sum(green_onion_scallion) AS green_onion_scallion'>,\n",
       " Column<'sum(grill) AS grill'>,\n",
       " Column<'sum(grill_barbecue) AS grill_barbecue'>,\n",
       " Column<'sum(ground_beef) AS ground_beef'>,\n",
       " Column<'sum(ground_lamb) AS ground_lamb'>,\n",
       " Column<'sum(guam) AS guam'>,\n",
       " Column<'sum(guava) AS guava'>,\n",
       " Column<'sum(haiti) AS haiti'>,\n",
       " Column<'sum(halibut) AS halibut'>,\n",
       " Column<'sum(halloween) AS halloween'>,\n",
       " Column<'sum(ham) AS ham'>,\n",
       " Column<'sum(hamburger) AS hamburger'>,\n",
       " Column<'sum(hanukkah) AS hanukkah'>,\n",
       " Column<'sum(harpercollins) AS harpercollins'>,\n",
       " Column<'sum(hawaii) AS hawaii'>,\n",
       " Column<'sum(hazelnut) AS hazelnut'>,\n",
       " Column<'sum(healdsburg) AS healdsburg'>,\n",
       " Column<'sum(healthy) AS healthy'>,\n",
       " Column<'sum(herb) AS herb'>,\n",
       " Column<'sum(high_fiber) AS high_fiber'>,\n",
       " Column<'sum(hollywood) AS hollywood'>,\n",
       " Column<'sum(hominy_cornmeal_masa) AS hominy_cornmeal_masa'>,\n",
       " Column<'sum(honey) AS honey'>,\n",
       " Column<'sum(honeydew) AS honeydew'>,\n",
       " Column<'sum(hors_doeuvre) AS hors_doeuvre'>,\n",
       " Column<'sum(horseradish) AS horseradish'>,\n",
       " Column<'sum(hot_drink) AS hot_drink'>,\n",
       " Column<'sum(hot_pepper) AS hot_pepper'>,\n",
       " Column<'sum(house_and_garden) AS house_and_garden'>,\n",
       " Column<'sum(house_cocktail) AS house_cocktail'>,\n",
       " Column<'sum(houston) AS houston'>,\n",
       " Column<'sum(hummus) AS hummus'>,\n",
       " Column<'sum(ice_cream) AS ice_cream'>,\n",
       " Column<'sum(ice_cream_machine) AS ice_cream_machine'>,\n",
       " Column<'sum(iced_coffee) AS iced_coffee'>,\n",
       " Column<'sum(iced_tea) AS iced_tea'>,\n",
       " Column<'sum(idaho) AS idaho'>,\n",
       " Column<'sum(illinois) AS illinois'>,\n",
       " Column<'sum(indiana) AS indiana'>,\n",
       " Column<'sum(iowa) AS iowa'>,\n",
       " Column<'sum(ireland) AS ireland'>,\n",
       " Column<'sum(israel) AS israel'>,\n",
       " Column<'sum(italy) AS italy'>,\n",
       " Column<'sum(jalapeño) AS `jalapeño`'>,\n",
       " Column<'sum(jam_or_jelly) AS jam_or_jelly'>,\n",
       " Column<'sum(jamaica) AS jamaica'>,\n",
       " Column<'sum(japan) AS japan'>,\n",
       " Column<'sum(jerusalem_artichoke) AS jerusalem_artichoke'>,\n",
       " Column<'sum(juicer) AS juicer'>,\n",
       " Column<'sum(jícama) AS `jícama`'>,\n",
       " Column<'sum(kahlúa) AS `kahlúa`'>,\n",
       " Column<'sum(kale) AS kale'>,\n",
       " Column<'sum(kansas) AS kansas'>,\n",
       " Column<'sum(kansas_city) AS kansas_city'>,\n",
       " Column<'sum(kentucky) AS kentucky'>,\n",
       " Column<'sum(kentucky_derby) AS kentucky_derby'>,\n",
       " Column<'sum(kid_friendly) AS kid_friendly'>,\n",
       " Column<'sum(kidney_friendly) AS kidney_friendly'>,\n",
       " Column<'sum(kirsch) AS kirsch'>,\n",
       " Column<'sum(kitchen_olympics) AS kitchen_olympics'>,\n",
       " Column<'sum(kiwi) AS kiwi'>,\n",
       " Column<'sum(kosher) AS kosher'>,\n",
       " Column<'sum(kosher_for_passover) AS kosher_for_passover'>,\n",
       " Column<'sum(kumquat) AS kumquat'>,\n",
       " Column<'sum(kwanzaa) AS kwanzaa'>,\n",
       " Column<'sum(labor_day) AS labor_day'>,\n",
       " Column<'sum(lamb) AS lamb'>,\n",
       " Column<'sum(lamb_chop) AS lamb_chop'>,\n",
       " Column<'sum(lamb_shank) AS lamb_shank'>,\n",
       " Column<'sum(lancaster) AS lancaster'>,\n",
       " Column<'sum(las_vegas) AS las_vegas'>,\n",
       " Column<'sum(lasagna) AS lasagna'>,\n",
       " Column<'sum(leafy_green) AS leafy_green'>,\n",
       " Column<'sum(leek) AS leek'>,\n",
       " Column<'sum(legume) AS legume'>,\n",
       " Column<'sum(lemon) AS lemon'>,\n",
       " Column<'sum(lemon_juice) AS lemon_juice'>,\n",
       " Column<'sum(lemongrass) AS lemongrass'>,\n",
       " Column<'sum(lentil) AS lentil'>,\n",
       " Column<'sum(lettuce) AS lettuce'>,\n",
       " Column<'sum(lima_bean) AS lima_bean'>,\n",
       " Column<'sum(lime) AS lime'>,\n",
       " Column<'sum(lime_juice) AS lime_juice'>,\n",
       " Column<'sum(lingonberry) AS lingonberry'>,\n",
       " Column<'sum(liqueur) AS liqueur'>,\n",
       " Column<'sum(lobster) AS lobster'>,\n",
       " Column<'sum(london) AS london'>,\n",
       " Column<'sum(long_beach) AS long_beach'>,\n",
       " Column<'sum(los_angeles) AS los_angeles'>,\n",
       " Column<'sum(louisiana) AS louisiana'>,\n",
       " Column<'sum(louisville) AS louisville'>,\n",
       " Column<'sum(low_cal) AS low_cal'>,\n",
       " Column<'sum(low_carb) AS low_carb'>,\n",
       " Column<'sum(low_cholesterol) AS low_cholesterol'>,\n",
       " Column<'sum(low_fat) AS low_fat'>,\n",
       " Column<'sum(low_sodium) AS low_sodium'>,\n",
       " Column<'sum(low_sugar) AS low_sugar'>,\n",
       " Column<'sum(low_no_sugar) AS low_no_sugar'>,\n",
       " Column<'sum(lunar_new_year) AS lunar_new_year'>,\n",
       " Column<'sum(lunch) AS lunch'>,\n",
       " Column<'sum(lychee) AS lychee'>,\n",
       " Column<'sum(macadamia_nut) AS macadamia_nut'>,\n",
       " Column<'sum(macaroni_and_cheese) AS macaroni_and_cheese'>,\n",
       " Column<'sum(maine) AS maine'>,\n",
       " Column<'sum(mandoline) AS mandoline'>,\n",
       " Column<'sum(mango) AS mango'>,\n",
       " Column<'sum(maple_syrup) AS maple_syrup'>,\n",
       " Column<'sum(mardi_gras) AS mardi_gras'>,\n",
       " Column<'sum(margarita) AS margarita'>,\n",
       " Column<'sum(marinade) AS marinade'>,\n",
       " Column<'sum(marinate) AS marinate'>,\n",
       " Column<'sum(marsala) AS marsala'>,\n",
       " Column<'sum(marscarpone) AS marscarpone'>,\n",
       " Column<'sum(marshmallow) AS marshmallow'>,\n",
       " Column<'sum(martini) AS martini'>,\n",
       " Column<'sum(maryland) AS maryland'>,\n",
       " Column<'sum(massachusetts) AS massachusetts'>,\n",
       " Column<'sum(mayonnaise) AS mayonnaise'>,\n",
       " Column<'sum(meat) AS meat'>,\n",
       " Column<'sum(meatball) AS meatball'>,\n",
       " Column<'sum(meatloaf) AS meatloaf'>,\n",
       " Column<'sum(melon) AS melon'>,\n",
       " Column<'sum(mexico) AS mexico'>,\n",
       " Column<'sum(mezcal) AS mezcal'>,\n",
       " Column<'sum(miami) AS miami'>,\n",
       " Column<'sum(michigan) AS michigan'>,\n",
       " Column<'sum(microwave) AS microwave'>,\n",
       " Column<'sum(midori) AS midori'>,\n",
       " Column<'sum(milk_cream) AS milk_cream'>,\n",
       " Column<'sum(minneapolis) AS minneapolis'>,\n",
       " Column<'sum(minnesota) AS minnesota'>,\n",
       " Column<'sum(mint) AS mint'>,\n",
       " Column<'sum(mississippi) AS mississippi'>,\n",
       " Column<'sum(missouri) AS missouri'>,\n",
       " Column<'sum(mixer) AS mixer'>,\n",
       " Column<'sum(molasses) AS molasses'>,\n",
       " Column<'sum(monterey_jack) AS monterey_jack'>,\n",
       " Column<'sum(mortar_and_pestle) AS mortar_and_pestle'>,\n",
       " Column<'sum(mothers_day) AS mothers_day'>,\n",
       " Column<'sum(mozzarella) AS mozzarella'>,\n",
       " Column<'sum(muffin) AS muffin'>,\n",
       " Column<'sum(mushroom) AS mushroom'>,\n",
       " Column<'sum(mussel) AS mussel'>,\n",
       " Column<'sum(mustard) AS mustard'>,\n",
       " Column<'sum(mustard_greens) AS mustard_greens'>,\n",
       " Column<'sum(nancy_silverton) AS nancy_silverton'>,\n",
       " Column<'sum(nebraska) AS nebraska'>,\n",
       " Column<'sum(nectarine) AS nectarine'>,\n",
       " Column<'sum(new_hampshire) AS new_hampshire'>,\n",
       " Column<'sum(new_jersey) AS new_jersey'>,\n",
       " Column<'sum(new_mexico) AS new_mexico'>,\n",
       " Column<'sum(new_orleans) AS new_orleans'>,\n",
       " Column<'sum(new_years_day) AS new_years_day'>,\n",
       " Column<'sum(new_years_eve) AS new_years_eve'>,\n",
       " Column<'sum(new_york) AS new_york'>,\n",
       " Column<'sum(no_meat_no_problem) AS no_meat_no_problem'>,\n",
       " Column<'sum(no_sugar_added) AS no_sugar_added'>,\n",
       " Column<'sum(no_cook) AS no_cook'>,\n",
       " Column<'sum(non_alcoholic) AS non_alcoholic'>,\n",
       " Column<'sum(noodle) AS noodle'>,\n",
       " Column<'sum(north_carolina) AS north_carolina'>,\n",
       " Column<'sum(nut) AS nut'>,\n",
       " Column<'sum(nutmeg) AS nutmeg'>,\n",
       " Column<'sum(oat) AS oat'>,\n",
       " Column<'sum(oatmeal) AS oatmeal'>,\n",
       " Column<'sum(octopus) AS octopus'>,\n",
       " Column<'sum(ohio) AS ohio'>,\n",
       " Column<'sum(oklahoma) AS oklahoma'>,\n",
       " Column<'sum(okra) AS okra'>,\n",
       " Column<'sum(oktoberfest) AS oktoberfest'>,\n",
       " Column<'sum(olive) AS olive'>,\n",
       " Column<'sum(omelet) AS omelet'>,\n",
       " Column<'sum(one_pot_meal) AS one_pot_meal'>,\n",
       " Column<'sum(onion) AS onion'>,\n",
       " Column<'sum(orange) AS orange'>,\n",
       " Column<'sum(orange_juice) AS orange_juice'>,\n",
       " Column<'sum(oregano) AS oregano'>,\n",
       " Column<'sum(oregon) AS oregon'>,\n",
       " Column<'sum(organic) AS organic'>,\n",
       " Column<'sum(orzo) AS orzo'>,\n",
       " Column<'sum(oscars) AS oscars'>,\n",
       " Column<'sum(oyster) AS oyster'>,\n",
       " Column<'sum(pacific_palisades) AS pacific_palisades'>,\n",
       " Column<'sum(paleo) AS paleo'>,\n",
       " Column<'sum(pan_fry) AS pan_fry'>,\n",
       " Column<'sum(pancake) AS pancake'>,\n",
       " Column<'sum(papaya) AS papaya'>,\n",
       " Column<'sum(paprika) AS paprika'>,\n",
       " Column<'sum(parade) AS parade'>,\n",
       " Column<'sum(paris) AS paris'>,\n",
       " Column<'sum(parmesan) AS parmesan'>,\n",
       " Column<'sum(parsley) AS parsley'>,\n",
       " Column<'sum(parsnip) AS parsnip'>,\n",
       " Column<'sum(party) AS party'>,\n",
       " Column<'sum(pasadena) AS pasadena'>,\n",
       " Column<'sum(passion_fruit) AS passion_fruit'>,\n",
       " Column<'sum(passover) AS passover'>,\n",
       " Column<'sum(pasta) AS pasta'>,\n",
       " Column<'sum(pasta_maker) AS pasta_maker'>,\n",
       " Column<'sum(pastry) AS pastry'>,\n",
       " Column<'sum(pea) AS pea'>,\n",
       " Column<'sum(peach) AS peach'>,\n",
       " Column<'sum(peanut) AS peanut'>,\n",
       " Column<'sum(peanut_butter) AS peanut_butter'>,\n",
       " Column<'sum(peanut_free) AS peanut_free'>,\n",
       " Column<'sum(pear) AS pear'>,\n",
       " Column<'sum(pecan) AS pecan'>,\n",
       " Column<'sum(pennsylvania) AS pennsylvania'>,\n",
       " Column<'sum(pepper) AS pepper'>,\n",
       " Column<'sum(pernod) AS pernod'>,\n",
       " Column<'sum(persian_new_year) AS persian_new_year'>,\n",
       " Column<'sum(persimmon) AS persimmon'>,\n",
       " Column<'sum(peru) AS peru'>,\n",
       " Column<'sum(pescatarian) AS pescatarian'>,\n",
       " Column<'sum(philippines) AS philippines'>,\n",
       " Column<'sum(phyllo_puff_pastry_dough) AS phyllo_puff_pastry_dough'>,\n",
       " Column<'sum(pickles) AS pickles'>,\n",
       " Column<'sum(picnic) AS picnic'>,\n",
       " Column<'sum(pie) AS pie'>,\n",
       " Column<'sum(pine_nut) AS pine_nut'>,\n",
       " Column<'sum(pineapple) AS pineapple'>,\n",
       " Column<'sum(pistachio) AS pistachio'>,\n",
       " Column<'sum(pittsburgh) AS pittsburgh'>,\n",
       " Column<'sum(pizza) AS pizza'>,\n",
       " Column<'sum(plantain) AS plantain'>,\n",
       " Column<'sum(plum) AS plum'>,\n",
       " Column<'sum(poach) AS poach'>,\n",
       " Column<'sum(poblano) AS poblano'>,\n",
       " Column<'sum(poker_game_night) AS poker_game_night'>,\n",
       " Column<'sum(pomegranate) AS pomegranate'>,\n",
       " Column<'sum(pomegranate_juice) AS pomegranate_juice'>,\n",
       " Column<'sum(poppy) AS poppy'>,\n",
       " Column<'sum(pork) AS pork'>,\n",
       " Column<'sum(pork_chop) AS pork_chop'>,\n",
       " Column<'sum(pork_rib) AS pork_rib'>,\n",
       " Column<'sum(pork_tenderloin) AS pork_tenderloin'>,\n",
       " Column<'sum(port) AS port'>,\n",
       " Column<'sum(portland) AS portland'>,\n",
       " Column<'sum(pot_pie) AS pot_pie'>,\n",
       " Column<'sum(potato) AS potato'>,\n",
       " Column<'sum(potato_salad) AS potato_salad'>,\n",
       " Column<'sum(potluck) AS potluck'>,\n",
       " Column<'sum(poultry) AS poultry'>,\n",
       " Column<'sum(poultry_sausage) AS poultry_sausage'>,\n",
       " Column<'sum(pressure_cooker) AS pressure_cooker'>,\n",
       " Column<'sum(prosciutto) AS prosciutto'>,\n",
       " Column<'sum(providence) AS providence'>,\n",
       " Column<'sum(prune) AS prune'>,\n",
       " Column<'sum(pumpkin) AS pumpkin'>,\n",
       " Column<'sum(punch) AS punch'>,\n",
       " Column<'sum(purim) AS purim'>,\n",
       " Column<'sum(quail) AS quail'>,\n",
       " Column<'sum(quiche) AS quiche'>,\n",
       " Column<'sum(quick_and_easy) AS quick_and_easy'>,\n",
       " Column<'sum(quick_and_healthy) AS quick_and_healthy'>,\n",
       " Column<'sum(quince) AS quince'>,\n",
       " Column<'sum(quinoa) AS quinoa'>,\n",
       " Column<'sum(rabbit) AS rabbit'>,\n",
       " Column<'sum(rack_of_lamb) AS rack_of_lamb'>,\n",
       " Column<'sum(radicchio) AS radicchio'>,\n",
       " Column<'sum(radish) AS radish'>,\n",
       " Column<'sum(raisin) AS raisin'>,\n",
       " Column<'sum(ramadan) AS ramadan'>,\n",
       " Column<'sum(ramekin) AS ramekin'>,\n",
       " Column<'sum(raspberry) AS raspberry'>,\n",
       " Column<'sum(raw) AS raw'>,\n",
       " Column<'sum(red_wine) AS red_wine'>,\n",
       " Column<'sum(rhode_island) AS rhode_island'>,\n",
       " Column<'sum(rhubarb) AS rhubarb'>,\n",
       " Column<'sum(rice) AS rice'>,\n",
       " Column<'sum(ricotta) AS ricotta'>,\n",
       " Column<'sum(roast) AS roast'>,\n",
       " Column<'sum(root_vegetable) AS root_vegetable'>,\n",
       " Column<'sum(rosemary) AS rosemary'>,\n",
       " Column<'sum(rosh_hashanah_yom_kippur) AS rosh_hashanah_yom_kippur'>,\n",
       " Column<'sum(rosé) AS `rosé`'>,\n",
       " Column<'sum(rub) AS rub'>,\n",
       " Column<'sum(rum) AS rum'>,\n",
       " Column<'sum(rutabaga) AS rutabaga'>,\n",
       " Column<'sum(rye) AS rye'>,\n",
       " Column<'sum(saffron) AS saffron'>,\n",
       " Column<'sum(sage) AS sage'>,\n",
       " Column<'sum(sake) AS sake'>,\n",
       " Column<'sum(salad) AS salad'>,\n",
       " Column<'sum(salad_dressing) AS salad_dressing'>,\n",
       " Column<'sum(salmon) AS salmon'>,\n",
       " Column<'sum(salsa) AS salsa'>,\n",
       " Column<'sum(san_francisco) AS san_francisco'>,\n",
       " Column<'sum(sandwich) AS sandwich'>,\n",
       " Column<'sum(sandwich_theory) AS sandwich_theory'>,\n",
       " Column<'sum(sangria) AS sangria'>,\n",
       " Column<'sum(santa_monica) AS santa_monica'>,\n",
       " Column<'sum(sardine) AS sardine'>,\n",
       " Column<'sum(sauce) AS sauce'>,\n",
       " Column<'sum(sausage) AS sausage'>,\n",
       " Column<'sum(sauté) AS `sauté`'>,\n",
       " Column<'sum(scallop) AS scallop'>,\n",
       " Column<'sum(scotch) AS scotch'>,\n",
       " Column<'sum(seafood) AS seafood'>,\n",
       " Column<'sum(seattle) AS seattle'>,\n",
       " Column<'sum(seed) AS seed'>,\n",
       " Column<'sum(self) AS self'>,\n",
       " Column<'sum(semolina) AS semolina'>,\n",
       " Column<'sum(sesame) AS sesame'>,\n",
       " Column<'sum(sesame_oil) AS sesame_oil'>,\n",
       " Column<'sum(shallot) AS shallot'>,\n",
       " Column<'sum(shavuot) AS shavuot'>,\n",
       " Column<'sum(shellfish) AS shellfish'>,\n",
       " Column<'sum(sherry) AS sherry'>,\n",
       " Column<'sum(shower) AS shower'>,\n",
       " Column<'sum(shrimp) AS shrimp'>,\n",
       " Column<'sum(side) AS side'>,\n",
       " Column<'sum(simmer) AS simmer'>,\n",
       " Column<'sum(skewer) AS skewer'>,\n",
       " Column<'sum(slow_cooker) AS slow_cooker'>,\n",
       " Column<'sum(smoker) AS smoker'>,\n",
       " Column<'sum(smoothie) AS smoothie'>,\n",
       " Column<'sum(snapper) AS snapper'>,\n",
       " Column<'sum(sorbet) AS sorbet'>,\n",
       " Column<'sum(soufflé_meringue) AS `soufflé_meringue`'>,\n",
       " Column<'sum(soup_stew) AS soup_stew'>,\n",
       " Column<'sum(sour_cream) AS sour_cream'>,\n",
       " Column<'sum(sourdough) AS sourdough'>,\n",
       " Column<'sum(south_carolina) AS south_carolina'>,\n",
       " Column<'sum(soy) AS soy'>,\n",
       " Column<'sum(soy_free) AS soy_free'>,\n",
       " Column<'sum(soy_sauce) AS soy_sauce'>,\n",
       " Column<'sum(spain) AS spain'>,\n",
       " Column<'sum(sparkling_wine) AS sparkling_wine'>,\n",
       " Column<'sum(spice) AS spice'>,\n",
       " Column<'sum(spinach) AS spinach'>,\n",
       " Column<'sum(spirit) AS spirit'>,\n",
       " Column<'sum(spring) AS spring'>,\n",
       " Column<'sum(spritzer) AS spritzer'>,\n",
       " Column<'sum(squash) AS squash'>,\n",
       " Column<'sum(squid) AS squid'>,\n",
       " Column<'sum(st_louis) AS st_louis'>,\n",
       " Column<'sum(st_patricks_day) AS st_patricks_day'>,\n",
       " Column<'sum(steak) AS steak'>,\n",
       " Column<'sum(steam) AS steam'>,\n",
       " Column<'sum(stew) AS stew'>,\n",
       " Column<'sum(stir_fry) AS stir_fry'>,\n",
       " Column<'sum(stock) AS stock'>,\n",
       " Column<'sum(strawberry) AS strawberry'>,\n",
       " Column<'sum(stuffing_dressing) AS stuffing_dressing'>,\n",
       " Column<'sum(sugar_conscious) AS sugar_conscious'>,\n",
       " Column<'sum(sugar_snap_pea) AS sugar_snap_pea'>,\n",
       " Column<'sum(sukkot) AS sukkot'>,\n",
       " Column<'sum(summer) AS summer'>,\n",
       " Column<'sum(super_bowl) AS super_bowl'>,\n",
       " Column<'sum(suzanne_goin) AS suzanne_goin'>,\n",
       " Column<'sum(sweet_potato_yam) AS sweet_potato_yam'>,\n",
       " Column<'sum(swiss_cheese) AS swiss_cheese'>,\n",
       " Column<'sum(switzerland) AS switzerland'>,\n",
       " Column<'sum(swordfish) AS swordfish'>,\n",
       " Column<'sum(taco) AS taco'>,\n",
       " Column<'sum(tailgating) AS tailgating'>,\n",
       " Column<'sum(tamarind) AS tamarind'>,\n",
       " Column<'sum(tangerine) AS tangerine'>,\n",
       " Column<'sum(tapioca) AS tapioca'>,\n",
       " Column<'sum(tarragon) AS tarragon'>,\n",
       " Column<'sum(tart) AS tart'>,\n",
       " Column<'sum(tea) AS tea'>,\n",
       " Column<'sum(tennessee) AS tennessee'>,\n",
       " Column<'sum(tequila) AS tequila'>,\n",
       " Column<'sum(tested_and_improved) AS tested_and_improved'>,\n",
       " Column<'sum(texas) AS texas'>,\n",
       " Column<'sum(thanksgiving) AS thanksgiving'>,\n",
       " Column<'sum(thyme) AS thyme'>,\n",
       " Column<'sum(tilapia) AS tilapia'>,\n",
       " Column<'sum(tofu) AS tofu'>,\n",
       " Column<'sum(tomatillo) AS tomatillo'>,\n",
       " Column<'sum(tomato) AS tomato'>,\n",
       " Column<'sum(tortillas) AS tortillas'>,\n",
       " Column<'sum(tree_nut) AS tree_nut'>,\n",
       " Column<'sum(tree_nut_free) AS tree_nut_free'>,\n",
       " Column<'sum(triple_sec) AS triple_sec'>,\n",
       " Column<'sum(tropical_fruit) AS tropical_fruit'>,\n",
       " Column<'sum(trout) AS trout'>,\n",
       " Column<'sum(tuna) AS tuna'>,\n",
       " Column<'sum(turnip) AS turnip'>,\n",
       " Column<'sum(utah) AS utah'>,\n",
       " Column<'sum(valentines_day) AS valentines_day'>,\n",
       " Column<'sum(vanilla) AS vanilla'>,\n",
       " Column<'sum(veal) AS veal'>,\n",
       " Column<'sum(vegan) AS vegan'>,\n",
       " Column<'sum(vegetable) AS vegetable'>,\n",
       " Column<'sum(vegetarian) AS vegetarian'>,\n",
       " Column<'sum(venison) AS venison'>,\n",
       " Column<'sum(vermont) AS vermont'>,\n",
       " Column<'sum(vermouth) AS vermouth'>,\n",
       " Column<'sum(vinegar) AS vinegar'>,\n",
       " Column<'sum(virginia) AS virginia'>,\n",
       " Column<'sum(vodka) AS vodka'>,\n",
       " Column<'sum(waffle) AS waffle'>,\n",
       " Column<'sum(walnut) AS walnut'>,\n",
       " Column<'sum(wasabi) AS wasabi'>,\n",
       " Column<'sum(washington) AS washington'>,\n",
       " Column<'sum(washington_dc) AS washington_dc'>,\n",
       " Column<'sum(watercress) AS watercress'>,\n",
       " Column<'sum(watermelon) AS watermelon'>,\n",
       " Column<'sum(wedding) AS wedding'>,\n",
       " Column<'sum(weelicious) AS weelicious'>,\n",
       " Column<'sum(west_virginia) AS west_virginia'>,\n",
       " Column<'sum(westwood) AS westwood'>,\n",
       " Column<'sum(wheat_gluten_free) AS wheat_gluten_free'>,\n",
       " Column<'sum(whiskey) AS whiskey'>,\n",
       " Column<'sum(white_wine) AS white_wine'>,\n",
       " Column<'sum(whole_wheat) AS whole_wheat'>,\n",
       " Column<'sum(wild_rice) AS wild_rice'>,\n",
       " Column<'sum(windsor) AS windsor'>,\n",
       " Column<'sum(wine) AS wine'>,\n",
       " Column<'sum(winter) AS winter'>,\n",
       " Column<'sum(wisconsin) AS wisconsin'>,\n",
       " Column<'sum(wok) AS wok'>,\n",
       " Column<'sum(yellow_squash) AS yellow_squash'>,\n",
       " Column<'sum(yogurt) AS yogurt'>,\n",
       " Column<'sum(yonkers) AS yonkers'>,\n",
       " Column<'sum(yuca) AS yuca'>,\n",
       " Column<'sum(zucchini) AS zucchini'>,\n",
       " Column<'sum(cookbooks) AS cookbooks'>,\n",
       " Column<'sum(leftovers) AS leftovers'>,\n",
       " Column<'sum(snack) AS snack'>,\n",
       " Column<'sum(snack_week) AS snack_week'>,\n",
       " Column<'sum(turkey) AS turkey'>]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inst_sum_of_binary_columns = list(map(lambda x:F.sum(F.col(x)).alias(x),BINARY_COLUMNS))\n",
    "inst_sum_of_binary_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "15ec7d95-d8ca-4ecc-9e55-0071d0443996",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_of_binary_columns = food.select(inst_sum_of_binary_columns)\n",
    "sum_of_binary_columns.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "83514e5e-0c9d-46a7-be77-a18118dc3cba",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pyspark.sql.types.Row'> <class 'pyspark.sql.dataframe.DataFrame'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'cakeweek': 6.0,\n",
       " 'wasteless': 1.0,\n",
       " '22_minute_meals': 17.0,\n",
       " '3_ingredient_recipes': 27.0,\n",
       " '30_days_of_groceries': 7.0,\n",
       " 'advance_prep_required': 109.0,\n",
       " 'alabama': 3.0,\n",
       " 'alaska': 2.0,\n",
       " 'alcoholic': 835.0,\n",
       " 'almond': 584.0,\n",
       " 'amaretto': 39.0,\n",
       " 'anchovy': 45.0,\n",
       " 'anise': 116.0,\n",
       " 'anniversary': 111.0,\n",
       " 'anthony_bourdain': 2.0,\n",
       " 'aperitif': 12.0,\n",
       " 'appetizer': 1372.0,\n",
       " 'apple': 666.0,\n",
       " 'apple_juice': 6.0,\n",
       " 'apricot': 269.0,\n",
       " 'arizona': 8.0,\n",
       " 'artichoke': 153.0,\n",
       " 'arugula': 278.0,\n",
       " 'asian_pear': 20.0,\n",
       " 'asparagus': 207.0,\n",
       " 'aspen': 1.0,\n",
       " 'atlanta': 7.0,\n",
       " 'australia': 7.0,\n",
       " 'avocado': 273.0,\n",
       " 'back_to_school': 202.0,\n",
       " 'backyard_bbq': 938.0,\n",
       " 'bacon': 587.0,\n",
       " 'bake': 4413.0,\n",
       " 'banana': 177.0,\n",
       " 'barley': 50.0,\n",
       " 'basil': 565.0,\n",
       " 'bass': 90.0,\n",
       " 'bastille_day': 30.0,\n",
       " 'bean': 577.0,\n",
       " 'beef': 811.0,\n",
       " 'beef_rib': 56.0,\n",
       " 'beef_shank': 14.0,\n",
       " 'beef_tenderloin': 72.0,\n",
       " 'beer': 112.0,\n",
       " 'beet': 209.0,\n",
       " 'bell_pepper': 767.0,\n",
       " 'berry': 465.0,\n",
       " 'beverly_hills': 1.0,\n",
       " 'birthday': 193.0,\n",
       " 'biscuit': 7.0,\n",
       " 'bitters': 127.0,\n",
       " 'blackberry': 144.0,\n",
       " 'blender': 617.0,\n",
       " 'blue_cheese': 235.0,\n",
       " 'blueberry': 180.0,\n",
       " 'boil': 445.0,\n",
       " 'bok_choy': 59.0,\n",
       " 'bon_appétit': 9354.0,\n",
       " 'bon_apptit': 10.0,\n",
       " 'boston': 5.0,\n",
       " 'bourbon': 125.0,\n",
       " 'braise': 330.0,\n",
       " 'bran': 3.0,\n",
       " 'brandy': 270.0,\n",
       " 'bread': 499.0,\n",
       " 'breadcrumbs': 55.0,\n",
       " 'breakfast': 712.0,\n",
       " 'brie': 12.0,\n",
       " 'brine': 36.0,\n",
       " 'brisket': 38.0,\n",
       " 'broccoli': 122.0,\n",
       " 'broccoli_rabe': 48.0,\n",
       " 'broil': 489.0,\n",
       " 'brooklyn': 7.0,\n",
       " 'brown_rice': 23.0,\n",
       " 'brownie': 3.0,\n",
       " 'brunch': 833.0,\n",
       " 'brussel_sprout': 56.0,\n",
       " 'buffalo': 8.0,\n",
       " 'buffet': 132.0,\n",
       " 'bulgaria': 1.0,\n",
       " 'bulgur': 30.0,\n",
       " 'burrito': 1.0,\n",
       " 'butter': 195.0,\n",
       " 'buttermilk': 73.0,\n",
       " 'butternut_squash': 182.0,\n",
       " 'butterscotch_caramel': 12.0,\n",
       " 'cabbage': 307.0,\n",
       " 'cake': 720.0,\n",
       " 'california': 146.0,\n",
       " 'calvados': 20.0,\n",
       " 'cambridge': 4.0,\n",
       " 'campari': 19.0,\n",
       " 'camping': 1.0,\n",
       " 'canada': 9.0,\n",
       " 'candy': 75.0,\n",
       " 'candy_thermometer': 66.0,\n",
       " 'cantaloupe': 51.0,\n",
       " 'capers': 185.0,\n",
       " 'caraway': 29.0,\n",
       " 'cardamom': 44.0,\n",
       " 'carrot': 701.0,\n",
       " 'cashew': 68.0,\n",
       " 'casserole_gratin': 199.0,\n",
       " 'cauliflower': 120.0,\n",
       " 'caviar': 1.0,\n",
       " 'celery': 341.0,\n",
       " 'chambord': 15.0,\n",
       " 'champagne': 106.0,\n",
       " 'chard': 85.0,\n",
       " 'chartreuse': 18.0,\n",
       " 'cheddar': 257.0,\n",
       " 'cheese': 1355.0,\n",
       " 'cherry': 244.0,\n",
       " 'chestnut': 74.0,\n",
       " 'chicago': 4.0,\n",
       " 'chicken': 1344.0,\n",
       " 'chickpea': 162.0,\n",
       " 'chile': 13.0,\n",
       " 'chile_pepper': 237.0,\n",
       " 'chili': 3.0,\n",
       " 'chill': 1126.0,\n",
       " 'chive': 251.0,\n",
       " 'chocolate': 855.0,\n",
       " 'christmas': 1039.0,\n",
       " 'christmas_eve': 331.0,\n",
       " 'cilantro': 607.0,\n",
       " 'cinco_de_mayo': 179.0,\n",
       " 'cinnamon': 464.0,\n",
       " 'citrus': 863.0,\n",
       " 'clam': 133.0,\n",
       " 'clove': 193.0,\n",
       " 'cobbler_crumble': 6.0,\n",
       " 'cocktail': 380.0,\n",
       " 'cocktail_party': 1154.0,\n",
       " 'coconut': 365.0,\n",
       " 'cod': 76.0,\n",
       " 'coffee': 190.0,\n",
       " 'coffee_grinder': 17.0,\n",
       " 'cognac_armagnac': 91.0,\n",
       " 'collard_greens': 31.0,\n",
       " 'colorado': 16.0,\n",
       " 'columbus': 3.0,\n",
       " 'condiment': 95.0,\n",
       " 'condiment_spread': 924.0,\n",
       " 'connecticut': 11.0,\n",
       " 'cook_like_a_diner': 4.0,\n",
       " 'cookbook_critic': 3.0,\n",
       " 'cookie': 69.0,\n",
       " 'cookies': 273.0,\n",
       " 'coriander': 131.0,\n",
       " 'corn': 371.0,\n",
       " 'cornmeal': 271.0,\n",
       " 'costa_mesa': 1.0,\n",
       " 'cottage_cheese': 37.0,\n",
       " 'couscous': 93.0,\n",
       " 'crab': 156.0,\n",
       " 'cranberry': 370.0,\n",
       " 'cranberry_sauce': 4.0,\n",
       " 'cream_cheese': 375.0,\n",
       " 'créme_de_cacao': 21.0,\n",
       " 'crêpe': 1.0,\n",
       " 'crme_de_cacao': 1.0,\n",
       " 'cuba': 3.0,\n",
       " 'cucumber': 378.0,\n",
       " 'cumin': 93.0,\n",
       " 'cupcake': 3.0,\n",
       " 'currant': 108.0,\n",
       " 'curry': 273.0,\n",
       " 'custard': 2.0,\n",
       " 'dairy': 1496.0,\n",
       " 'dairy_free': 3206.0,\n",
       " 'dallas': 5.0,\n",
       " 'date': 123.0,\n",
       " 'deep_fry': 92.0,\n",
       " 'denver': 3.0,\n",
       " 'digestif': 5.0,\n",
       " 'dill': 271.0,\n",
       " 'dinner': 2704.0,\n",
       " 'dip': 20.0,\n",
       " 'diwali': 25.0,\n",
       " 'dominican_republic': 2.0,\n",
       " 'dorie_greenspan': 1.0,\n",
       " 'double_boiler': 32.0,\n",
       " 'dried_fruit': 206.0,\n",
       " 'drink': 1087.0,\n",
       " 'drinks': 11.0,\n",
       " 'duck': 99.0,\n",
       " 'easter': 234.0,\n",
       " 'eau_de_vie': 9.0,\n",
       " 'edible_gift': 120.0,\n",
       " 'egg': 1768.0,\n",
       " 'egg_nog': 9.0,\n",
       " 'eggplant': 219.0,\n",
       " 'egypt': 1.0,\n",
       " 'emeril_lagasse': 1.0,\n",
       " 'endive': 112.0,\n",
       " 'engagement_party': 183.0,\n",
       " 'england': 1.0,\n",
       " 'entertaining': 8.0,\n",
       " 'epi__ushg': 2.0,\n",
       " 'epi_loves_the_microwave': 2.0,\n",
       " 'escarole': 43.0,\n",
       " 'fall': 3015.0,\n",
       " 'family_reunion': 316.0,\n",
       " 'fat_free': 376.0,\n",
       " 'fathers_day': 295.0,\n",
       " 'fennel': 371.0,\n",
       " 'feta': 229.0,\n",
       " 'fig': 154.0,\n",
       " 'fish': 936.0,\n",
       " 'flaming_hot_summer': 12.0,\n",
       " 'flat_bread': 4.0,\n",
       " 'florida': 43.0,\n",
       " 'fontina': 40.0,\n",
       " 'food_processor': 972.0,\n",
       " 'fortified_wine': 75.0,\n",
       " 'fourth_of_july': 376.0,\n",
       " 'france': 13.0,\n",
       " 'frangelico': 23.0,\n",
       " 'frankenrecipe': 1.0,\n",
       " 'freeze_chill': 417.0,\n",
       " 'freezer_food': 3.0,\n",
       " 'friendsgiving': 9.0,\n",
       " 'frittata': 2.0,\n",
       " 'fritter': 1.0,\n",
       " 'frozen_dessert': 354.0,\n",
       " 'fruit': 1958.0,\n",
       " 'fruit_juice': 488.0,\n",
       " 'fry': 430.0,\n",
       " 'game': 36.0,\n",
       " 'garlic': 1643.0,\n",
       " 'georgia': 20.0,\n",
       " 'germany': 1.0,\n",
       " 'gin': 226.0,\n",
       " 'ginger': 1146.0,\n",
       " 'goat_cheese': 307.0,\n",
       " 'goose': 16.0,\n",
       " 'gouda': 10.0,\n",
       " 'gourmet': 6647.0,\n",
       " 'graduation': 89.0,\n",
       " 'grains': 9.0,\n",
       " 'grand_marnier': 1.0,\n",
       " 'granola': 5.0,\n",
       " 'grape': 70.0,\n",
       " 'grapefruit': 120.0,\n",
       " 'grappa': 4.0,\n",
       " 'green_bean': 226.0,\n",
       " 'green_onion_scallion': 116.0,\n",
       " 'grill': 564.0,\n",
       " 'grill_barbecue': 1104.0,\n",
       " 'ground_beef': 90.0,\n",
       " 'ground_lamb': 35.0,\n",
       " 'guam': 1.0,\n",
       " 'guava': 21.0,\n",
       " 'haiti': 1.0,\n",
       " 'halibut': 89.0,\n",
       " 'halloween': 42.0,\n",
       " 'ham': 222.0,\n",
       " 'hamburger': 4.0,\n",
       " 'hanukkah': 118.0,\n",
       " 'harpercollins': 121.0,\n",
       " 'hawaii': 4.0,\n",
       " 'hazelnut': 182.0,\n",
       " 'healdsburg': 2.0,\n",
       " 'healthy': 2350.0,\n",
       " 'herb': 1681.0,\n",
       " 'high_fiber': 1012.0,\n",
       " 'hollywood': 1.0,\n",
       " 'hominy_cornmeal_masa': 84.0,\n",
       " 'honey': 401.0,\n",
       " 'honeydew': 40.0,\n",
       " 'hors_doeuvre': 17.0,\n",
       " 'horseradish': 166.0,\n",
       " 'hot_drink': 37.0,\n",
       " 'hot_pepper': 487.0,\n",
       " 'house_and_garden': 459.0,\n",
       " 'house_cocktail': 6.0,\n",
       " 'houston': 1.0,\n",
       " 'hummus': 2.0,\n",
       " 'ice_cream': 12.0,\n",
       " 'ice_cream_machine': 162.0,\n",
       " 'iced_coffee': 2.0,\n",
       " 'iced_tea': 18.0,\n",
       " 'idaho': 1.0,\n",
       " 'illinois': 8.0,\n",
       " 'indiana': 5.0,\n",
       " 'iowa': 1.0,\n",
       " 'ireland': 14.0,\n",
       " 'israel': 2.0,\n",
       " 'italy': 6.0,\n",
       " 'jalapeño': 361.0,\n",
       " 'jam_or_jelly': 159.0,\n",
       " 'jamaica': 1.0,\n",
       " 'japan': 2.0,\n",
       " 'jerusalem_artichoke': 19.0,\n",
       " 'juicer': 6.0,\n",
       " 'jícama': 36.0,\n",
       " 'kahlúa': 25.0,\n",
       " 'kale': 145.0,\n",
       " 'kansas': 3.0,\n",
       " 'kansas_city': 2.0,\n",
       " 'kentucky': 3.0,\n",
       " 'kentucky_derby': 40.0,\n",
       " 'kid_friendly': 1791.0,\n",
       " 'kidney_friendly': 2313.0,\n",
       " 'kirsch': 31.0,\n",
       " 'kitchen_olympics': 1.0,\n",
       " 'kiwi': 34.0,\n",
       " 'kosher': 6175.0,\n",
       " 'kosher_for_passover': 31.0,\n",
       " 'kumquat': 43.0,\n",
       " 'kwanzaa': 13.0,\n",
       " 'labor_day': 9.0,\n",
       " 'lamb': 359.0,\n",
       " 'lamb_chop': 45.0,\n",
       " 'lamb_shank': 18.0,\n",
       " 'lancaster': 4.0,\n",
       " 'las_vegas': 1.0,\n",
       " 'lasagna': 3.0,\n",
       " 'leafy_green': 902.0,\n",
       " 'leek': 266.0,\n",
       " 'legume': 69.0,\n",
       " 'lemon': 1104.0,\n",
       " 'lemon_juice': 154.0,\n",
       " 'lemongrass': 83.0,\n",
       " 'lentil': 98.0,\n",
       " 'lettuce': 225.0,\n",
       " 'lima_bean': 55.0,\n",
       " 'lime': 546.0,\n",
       " 'lime_juice': 133.0,\n",
       " 'lingonberry': 5.0,\n",
       " 'liqueur': 317.0,\n",
       " 'lobster': 83.0,\n",
       " 'london': 1.0,\n",
       " 'long_beach': 2.0,\n",
       " 'los_angeles': 21.0,\n",
       " 'louisiana': 4.0,\n",
       " 'louisville': 2.0,\n",
       " 'low_cal': 1048.0,\n",
       " 'low_carb': 363.0,\n",
       " 'low_cholesterol': 444.0,\n",
       " 'low_fat': 929.0,\n",
       " 'low_sodium': 389.0,\n",
       " 'low_sugar': 14.0,\n",
       " 'low_no_sugar': 282.0,\n",
       " 'lunar_new_year': 46.0,\n",
       " 'lunch': 1359.0,\n",
       " 'lychee': 8.0,\n",
       " 'macadamia_nut': 50.0,\n",
       " 'macaroni_and_cheese': 3.0,\n",
       " 'maine': 5.0,\n",
       " 'mandoline': 10.0,\n",
       " 'mango': 244.0,\n",
       " 'maple_syrup': 146.0,\n",
       " 'mardi_gras': 59.0,\n",
       " 'margarita': 20.0,\n",
       " 'marinade': 1.0,\n",
       " 'marinate': 601.0,\n",
       " 'marsala': 36.0,\n",
       " 'marscarpone': 6.0,\n",
       " 'marshmallow': 9.0,\n",
       " 'martini': 7.0,\n",
       " 'maryland': 12.0,\n",
       " 'massachusetts': 47.0,\n",
       " 'mayonnaise': 378.0,\n",
       " 'meat': 285.0,\n",
       " 'meatball': 8.0,\n",
       " 'meatloaf': 2.0,\n",
       " 'melon': 62.0,\n",
       " 'mexico': 5.0,\n",
       " 'mezcal': 1.0,\n",
       " 'miami': 2.0,\n",
       " 'michigan': 24.0,\n",
       " 'microwave': 37.0,\n",
       " 'midori': 6.0,\n",
       " 'milk_cream': 1994.0,\n",
       " 'minneapolis': 2.0,\n",
       " 'minnesota': 11.0,\n",
       " 'mint': 615.0,\n",
       " 'mississippi': 1.0,\n",
       " 'missouri': 14.0,\n",
       " 'mixer': 630.0,\n",
       " 'molasses': 112.0,\n",
       " 'monterey_jack': 50.0,\n",
       " 'mortar_and_pestle': 2.0,\n",
       " 'mothers_day': 165.0,\n",
       " 'mozzarella': 190.0,\n",
       " 'muffin': 5.0,\n",
       " 'mushroom': 843.0,\n",
       " 'mussel': 126.0,\n",
       " 'mustard': 600.0,\n",
       " 'mustard_greens': 36.0,\n",
       " 'nancy_silverton': 1.0,\n",
       " 'nebraska': 3.0,\n",
       " 'nectarine': 60.0,\n",
       " 'new_hampshire': 1.0,\n",
       " 'new_jersey': 25.0,\n",
       " 'new_mexico': 15.0,\n",
       " 'new_orleans': 2.0,\n",
       " 'new_years_day': 82.0,\n",
       " 'new_years_eve': 165.0,\n",
       " 'new_york': 49.0,\n",
       " 'no_meat_no_problem': 6.0,\n",
       " 'no_sugar_added': 3132.0,\n",
       " 'no_cook': 1251.0,\n",
       " 'non_alcoholic': 218.0,\n",
       " 'noodle': 162.0,\n",
       " 'north_carolina': 17.0,\n",
       " 'nut': 904.0,\n",
       " 'nutmeg': 108.0,\n",
       " 'oat': 137.0,\n",
       " 'oatmeal': 12.0,\n",
       " 'octopus': 7.0,\n",
       " 'ohio': 13.0,\n",
       " 'oklahoma': 4.0,\n",
       " 'okra': 27.0,\n",
       " 'oktoberfest': 37.0,\n",
       " 'olive': 556.0,\n",
       " 'omelet': 2.0,\n",
       " 'one_pot_meal': 11.0,\n",
       " 'onion': 2238.0,\n",
       " 'orange': 837.0,\n",
       " 'orange_juice': 52.0,\n",
       " 'oregano': 101.0,\n",
       " 'oregon': 19.0,\n",
       " 'organic': 7.0,\n",
       " 'orzo': 2.0,\n",
       " 'oscars': 112.0,\n",
       " 'oyster': 38.0,\n",
       " 'pacific_palisades': 1.0,\n",
       " 'paleo': 779.0,\n",
       " 'pan_fry': 249.0,\n",
       " 'pancake': 5.0,\n",
       " 'papaya': 58.0,\n",
       " 'paprika': 55.0,\n",
       " 'parade': 143.0,\n",
       " 'paris': 4.0,\n",
       " 'parmesan': 644.0,\n",
       " 'parsley': 514.0,\n",
       " 'parsnip': 140.0,\n",
       " 'party': 892.0,\n",
       " 'pasadena': 12.0,\n",
       " 'passion_fruit': 44.0,\n",
       " 'passover': 183.0,\n",
       " 'pasta': 968.0,\n",
       " 'pasta_maker': 12.0,\n",
       " 'pastry': 99.0,\n",
       " 'pea': 211.0,\n",
       " 'peach': 194.0,\n",
       " 'peanut': 153.0,\n",
       " 'peanut_butter': 32.0,\n",
       " 'peanut_free': 8390.0,\n",
       " 'pear': 279.0,\n",
       " 'pecan': 331.0,\n",
       " 'pennsylvania': 34.0,\n",
       " 'pepper': 612.0,\n",
       " 'pernod': 54.0,\n",
       " 'persian_new_year': 4.0,\n",
       " 'persimmon': 11.0,\n",
       " 'peru': 2.0,\n",
       " 'pescatarian': 6042.0,\n",
       " 'philippines': 5.0,\n",
       " 'phyllo_puff_pastry_dough': 126.0,\n",
       " 'pickles': 16.0,\n",
       " 'picnic': 519.0,\n",
       " 'pie': 21.0,\n",
       " 'pine_nut': 227.0,\n",
       " 'pineapple': 264.0,\n",
       " 'pistachio': 176.0,\n",
       " 'pittsburgh': 12.0,\n",
       " 'pizza': 10.0,\n",
       " 'plantain': 21.0,\n",
       " 'plum': 114.0,\n",
       " 'poach': 135.0,\n",
       " 'poblano': 7.0,\n",
       " 'poker_game_night': 135.0,\n",
       " 'pomegranate': 77.0,\n",
       " 'pomegranate_juice': 7.0,\n",
       " 'poppy': 20.0,\n",
       " 'pork': 928.0,\n",
       " 'pork_chop': 93.0,\n",
       " 'pork_rib': 57.0,\n",
       " 'pork_tenderloin': 120.0,\n",
       " 'port': 110.0,\n",
       " 'portland': 15.0,\n",
       " 'pot_pie': 2.0,\n",
       " 'potato': 1128.0,\n",
       " 'potato_salad': 2.0,\n",
       " 'potluck': 485.0,\n",
       " 'poultry': 620.0,\n",
       " 'poultry_sausage': 23.0,\n",
       " 'pressure_cooker': 8.0,\n",
       " 'prosciutto': 155.0,\n",
       " 'providence': 1.0,\n",
       " 'prune': 108.0,\n",
       " 'pumpkin': 118.0,\n",
       " 'punch': 73.0,\n",
       " 'purim': 13.0,\n",
       " 'quail': 15.0,\n",
       " 'quiche': 1.0,\n",
       " 'quick_and_easy': 5372.0,\n",
       " 'quick_and_healthy': 65.0,\n",
       " 'quince': 31.0,\n",
       " 'quinoa': 51.0,\n",
       " 'rabbit': 7.0,\n",
       " 'rack_of_lamb': 29.0,\n",
       " 'radicchio': 18.0,\n",
       " 'radish': 208.0,\n",
       " 'raisin': 340.0,\n",
       " 'ramadan': 39.0,\n",
       " 'ramekin': 34.0,\n",
       " 'raspberry': 340.0,\n",
       " 'raw': 77.0,\n",
       " 'red_wine': 271.0,\n",
       " 'rhode_island': 8.0,\n",
       " 'rhubarb': 80.0,\n",
       " 'rice': 538.0,\n",
       " 'ricotta': 160.0,\n",
       " 'roast': 1320.0,\n",
       " 'root_vegetable': 187.0,\n",
       " 'rosemary': 370.0,\n",
       " 'rosh_hashanah_yom_kippur': 68.0,\n",
       " 'rosé': 4.0,\n",
       " 'rub': 3.0,\n",
       " 'rum': 331.0,\n",
       " 'rutabaga': 44.0,\n",
       " 'rye': 19.0,\n",
       " 'saffron': 112.0,\n",
       " 'sage': 182.0,\n",
       " 'sake': 21.0,\n",
       " 'salad': 1515.0,\n",
       " 'salad_dressing': 178.0,\n",
       " 'salmon': 315.0,\n",
       " 'salsa': 6.0,\n",
       " 'san_francisco': 8.0,\n",
       " 'sandwich': 428.0,\n",
       " 'sandwich_theory': 13.0,\n",
       " 'sangria': 17.0,\n",
       " 'santa_monica': 10.0,\n",
       " 'sardine': 4.0,\n",
       " 'sauce': 1065.0,\n",
       " 'sausage': 384.0,\n",
       " 'sauté': 2044.0,\n",
       " 'scallop': 170.0,\n",
       " 'scotch': 22.0,\n",
       " 'seafood': 401.0,\n",
       " 'seattle': 10.0,\n",
       " 'seed': 150.0,\n",
       " 'self': 312.0,\n",
       " 'semolina': 15.0,\n",
       " 'sesame': 147.0,\n",
       " 'sesame_oil': 42.0,\n",
       " 'shallot': 353.0,\n",
       " 'shavuot': 21.0,\n",
       " 'shellfish': 559.0,\n",
       " 'sherry': 125.0,\n",
       " 'shower': 327.0,\n",
       " 'shrimp': 504.0,\n",
       " 'side': 3151.0,\n",
       " 'simmer': 889.0,\n",
       " 'skewer': 7.0,\n",
       " 'slow_cooker': 25.0,\n",
       " 'smoker': 7.0,\n",
       " 'smoothie': 42.0,\n",
       " 'snapper': 59.0,\n",
       " 'sorbet': 1.0,\n",
       " 'soufflé_meringue': 3.0,\n",
       " 'soup_stew': 1163.0,\n",
       " 'sour_cream': 404.0,\n",
       " 'sourdough': 1.0,\n",
       " 'south_carolina': 10.0,\n",
       " 'soy': 110.0,\n",
       " 'soy_free': 8088.0,\n",
       " 'soy_sauce': 293.0,\n",
       " 'spain': 2.0,\n",
       " 'sparkling_wine': 54.0,\n",
       " 'spice': 680.0,\n",
       " 'spinach': 377.0,\n",
       " 'spirit': 219.0,\n",
       " 'spring': 1715.0,\n",
       " 'spritzer': 7.0,\n",
       " 'squash': 151.0,\n",
       " 'squid': 46.0,\n",
       " 'st_louis': 7.0,\n",
       " 'st_patricks_day': 55.0,\n",
       " 'steak': 243.0,\n",
       " 'steam': 225.0,\n",
       " 'stew': 178.0,\n",
       " 'stir_fry': 227.0,\n",
       " 'stock': 3.0,\n",
       " 'strawberry': 308.0,\n",
       " 'stuffing_dressing': 138.0,\n",
       " 'sugar_conscious': 2466.0,\n",
       " 'sugar_snap_pea': 77.0,\n",
       " 'sukkot': 14.0,\n",
       " 'summer': 4150.0,\n",
       " 'super_bowl': 311.0,\n",
       " 'suzanne_goin': 1.0,\n",
       " 'sweet_potato_yam': 286.0,\n",
       " 'swiss_cheese': 74.0,\n",
       " 'switzerland': 4.0,\n",
       " 'swordfish': 16.0,\n",
       " 'taco': 8.0,\n",
       " 'tailgating': 129.0,\n",
       " 'tamarind': 21.0,\n",
       " 'tangerine': 36.0,\n",
       " 'tapioca': 14.0,\n",
       " 'tarragon': 240.0,\n",
       " 'tart': 8.0,\n",
       " 'tea': 110.0,\n",
       " 'tennessee': 2.0,\n",
       " 'tequila': 89.0,\n",
       " 'tested_and_improved': 1.0,\n",
       " 'texas': 23.0,\n",
       " 'thanksgiving': 1458.0,\n",
       " 'thyme': 333.0,\n",
       " 'tilapia': 12.0,\n",
       " 'tofu': 92.0,\n",
       " 'tomatillo': 82.0,\n",
       " 'tomato': 2140.0,\n",
       " 'tortillas': 158.0,\n",
       " 'tree_nut': 106.0,\n",
       " 'tree_nut_free': 7044.0,\n",
       " 'triple_sec': 80.0,\n",
       " 'tropical_fruit': 62.0,\n",
       " 'trout': 54.0,\n",
       " 'tuna': 134.0,\n",
       " 'turnip': 92.0,\n",
       " 'utah': 3.0,\n",
       " 'valentines_day': 232.0,\n",
       " 'vanilla': 267.0,\n",
       " 'veal': 123.0,\n",
       " 'vegan': 1848.0,\n",
       " 'vegetable': 2087.0,\n",
       " 'vegetarian': 6844.0,\n",
       " 'venison': 9.0,\n",
       " 'vermont': 5.0,\n",
       " 'vermouth': 11.0,\n",
       " 'vinegar': 657.0,\n",
       " 'virginia': 11.0,\n",
       " 'vodka': 125.0,\n",
       " 'waffle': 1.0,\n",
       " 'walnut': 425.0,\n",
       " 'wasabi': 23.0,\n",
       " 'washington': 26.0,\n",
       " 'washington_dc': 10.0,\n",
       " 'watercress': 138.0,\n",
       " 'watermelon': 72.0,\n",
       " 'wedding': 112.0,\n",
       " 'weelicious': 33.0,\n",
       " 'west_virginia': 1.0,\n",
       " 'westwood': 1.0,\n",
       " 'wheat_gluten_free': 4906.0,\n",
       " 'whiskey': 57.0,\n",
       " 'white_wine': 487.0,\n",
       " 'whole_wheat': 30.0,\n",
       " 'wild_rice': 18.0,\n",
       " 'windsor': 1.0,\n",
       " 'wine': 191.0,\n",
       " 'winter': 3099.0,\n",
       " 'wisconsin': 4.0,\n",
       " 'wok': 102.0,\n",
       " 'yellow_squash': 25.0,\n",
       " 'yogurt': 528.0,\n",
       " 'yonkers': 1.0,\n",
       " 'yuca': 6.0,\n",
       " 'zucchini': 298.0,\n",
       " 'cookbooks': 3.0,\n",
       " 'leftovers': 7.0,\n",
       " 'snack': 28.0,\n",
       " 'snack_week': 19.0,\n",
       " 'turkey': 456.0}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(type(sum_of_binary_columns.head()),type(sum_of_binary_columns))\n",
    "sum_of_binary_columns = sum_of_binary_columns.head().asDict()\n",
    "sum_of_binary_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fae43567-b99e-4001-aed2-a93874a6a8e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_rows = food.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2d37f79c-f88d-4d72-b142-8afceef405cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "167"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "too_rare_features = [k for k,v in sum_of_binary_columns.items() if v<10 or v>num_rows-10]\n",
    "len(too_rare_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "06e5cfa2-eb19-465f-b29f-57f6ee91f103",
   "metadata": {},
   "outputs": [],
   "source": [
    "BINARY_COLUMNS = set(BINARY_COLUMNS) - set(too_rare_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1066bdd2-e5c6-4a80-b511-82d50432408b",
   "metadata": {},
   "outputs": [],
   "source": [
    "food = (\n",
    "    food.withColumn(\"protein_ratio\",F.col(\"protein\")*4/F.col(\"calories\"))\n",
    "    .withColumn(\"fat_ratio\",F.col(\"fat\")*9/F.col(\"calories\"))\n",
    ")\n",
    "ratio_set=[\"protein_ratio\",\"fat_ratio\"]\n",
    "food = food.fillna(0.0,subset=ratio_set)\n",
    "CONTINUOUS_COLUMNS.extend(ratio_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8065ded9-7b3b-428e-9c41-3a12182b8798",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20048 7\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "\n",
    "continuous_features = VectorAssembler(\n",
    "    inputCols = CONTINUOUS_COLUMNS,\n",
    "    outputCol  = \"continuous_features\"\n",
    ")\n",
    "vector_food = food.select(CONTINUOUS_COLUMNS)\n",
    "print(vector_food.count(),len(vector_food.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a08f5b4e-7046-4eaa-8101-45ff26238052",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for x in CONTINUOUS_COLUMNS:\n",
    "#     vector_food = vector_food.where(~F.isnull(F.col(x)))\n",
    "# print(vector_food.count(),len(vector_food.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "07f5658f-0aa5-4be4-a1fe-07c0cc2b3147",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15861 7\n"
     ]
    }
   ],
   "source": [
    "vector_food = vector_food.dropna(how=\"any\")\n",
    "print(vector_food.count(),len(vector_food.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "64baf549-52bf-4bcd-9d18-9c692f05a11e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+--------+-------+----+------+--------------------+-------------------+---------------------------------------------------------------------+\n",
      "|rating|calories|protein|fat |sodium|protein_ratio       |fat_ratio          |continuous_features                                                  |\n",
      "+------+--------+-------+----+------+--------------------+-------------------+---------------------------------------------------------------------+\n",
      "|2.5   |426.0   |30.0   |7.0 |559.0 |0.28169014084507044 |0.14788732394366197|[2.5,426.0,30.0,7.0,559.0,0.28169014084507044,0.14788732394366197]   |\n",
      "|4.375 |403.0   |18.0   |23.0|1439.0|0.17866004962779156 |0.5136476426799007 |[4.375,403.0,18.0,23.0,1439.0,0.17866004962779156,0.5136476426799007]|\n",
      "|3.75  |165.0   |6.0    |7.0 |165.0 |0.14545454545454545 |0.38181818181818183|[3.75,165.0,6.0,7.0,165.0,0.14545454545454545,0.38181818181818183]   |\n",
      "|3.125 |547.0   |20.0   |32.0|452.0 |0.14625228519195613 |0.526508226691042  |[3.125,547.0,20.0,32.0,452.0,0.14625228519195613,0.526508226691042]  |\n",
      "|4.375 |948.0   |19.0   |79.0|1042.0|0.08016877637130802 |0.75               |[4.375,948.0,19.0,79.0,1042.0,0.08016877637130802,0.75]              |\n",
      "|4.375 |170.0   |7.0    |10.0|1272.0|0.16470588235294117 |0.5294117647058824 |[4.375,170.0,7.0,10.0,1272.0,0.16470588235294117,0.5294117647058824] |\n",
      "|3.75  |602.0   |23.0   |41.0|1696.0|0.15282392026578073 |0.6129568106312292 |[3.75,602.0,23.0,41.0,1696.0,0.15282392026578073,0.6129568106312292] |\n",
      "|3.75  |256.0   |4.0    |5.0 |30.0  |0.0625              |0.17578125         |[3.75,256.0,4.0,5.0,30.0,0.0625,0.17578125]                          |\n",
      "|4.375 |766.0   |12.0   |48.0|439.0 |0.06266318537859007 |0.5639686684073107 |[4.375,766.0,12.0,48.0,439.0,0.06266318537859007,0.5639686684073107] |\n",
      "|4.375 |174.0   |11.0   |12.0|176.0 |0.25287356321839083 |0.6206896551724138 |[4.375,174.0,11.0,12.0,176.0,0.25287356321839083,0.6206896551724138] |\n",
      "|3.125 |134.0   |4.0    |3.0 |1394.0|0.11940298507462686 |0.20149253731343283|[3.125,134.0,4.0,3.0,1394.0,0.11940298507462686,0.20149253731343283] |\n",
      "|4.375 |382.0   |5.0    |31.0|977.0 |0.05235602094240838 |0.7303664921465969 |[4.375,382.0,5.0,31.0,977.0,0.05235602094240838,0.7303664921465969]  |\n",
      "|1.875 |146.0   |4.0    |5.0 |160.0 |0.1095890410958904  |0.3082191780821918 |[1.875,146.0,4.0,5.0,160.0,0.1095890410958904,0.3082191780821918]    |\n",
      "|4.375 |890.0   |59.0   |68.0|1027.0|0.2651685393258427  |0.6876404494382022 |[4.375,890.0,59.0,68.0,1027.0,0.2651685393258427,0.6876404494382022] |\n",
      "|5.0   |107.0   |5.0    |7.0 |344.0 |0.18691588785046728 |0.5887850467289719 |[5.0,107.0,5.0,7.0,344.0,0.18691588785046728,0.5887850467289719]     |\n",
      "|5.0   |421.0   |10.0   |33.0|383.0 |0.09501187648456057 |0.7054631828978623 |[5.0,421.0,10.0,33.0,383.0,0.09501187648456057,0.7054631828978623]   |\n",
      "|4.375 |345.0   |11.0   |19.0|423.0 |0.12753623188405797 |0.4956521739130435 |[4.375,345.0,11.0,19.0,423.0,0.12753623188405797,0.4956521739130435] |\n",
      "|3.75  |279.0   |3.0    |30.0|206.0 |0.043010752688172046|0.967741935483871  |[3.75,279.0,3.0,30.0,206.0,0.043010752688172046,0.967741935483871]   |\n",
      "|0.0   |95.0    |1.0    |7.0 |103.0 |0.042105263157894736|0.6631578947368421 |[0.0,95.0,1.0,7.0,103.0,0.042105263157894736,0.6631578947368421]     |\n",
      "|3.75  |215.0   |6.0    |20.0|250.0 |0.11162790697674418 |0.8372093023255814 |[3.75,215.0,6.0,20.0,250.0,0.11162790697674418,0.8372093023255814]   |\n",
      "+------+--------+-------+----+------+--------------------+-------------------+---------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vector_variable = continuous_features.transform(vector_food)\n",
    "vector_variable.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "27846601-ebdd-4e3b-bbae-31eeda5e5a23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- pearson(continuous_features): matrix (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.stat import Correlation\n",
    "\n",
    "correlation = Correlation.corr(vector_variable,\"continuous_features\")\n",
    "correlation.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "97cf70f9-a554-4dda-a0d4-589d4f195ad9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|pearson(continuous_features)                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |\n",
      "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|1.0                  0.10245366771571467  0.11329164938207341  ... (7 total)\\n0.10245366771571467  1.0                  0.7582403836435118   ...\\n0.11329164938207341  0.7582403836435118   1.0                  ...\\n0.11153613940572367  0.9181251631300806   0.664898601882207    ...\\n0.06527344745367157  0.5169638229319906   0.5855234361086401   ...\\n0.09443879704546937  0.16550588587310078  0.6004673118081287   ...\\n0.12994741969799842  0.17785614447847162  0.10962704968896074  ...|\n",
      "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "correlation.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8657e422-f5c1-450a-bd25-48d6372bec9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(pearson(continuous_features)=DenseMatrix(7, 7, [1.0, 0.1025, 0.1133, 0.1115, 0.0653, 0.0944, 0.1299, 0.1025, ..., 0.0249, 0.1299, 0.1779, 0.1096, 0.4257, 0.0342, 0.0249, 1.0], False))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correlation.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "61af041d-2be5-4c0e-9290-918502cf84e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseMatrix(7, 7, [1.0, 0.1025, 0.1133, 0.1115, 0.0653, 0.0944, 0.1299, 0.1025, ..., 0.0249, 0.1299, 0.1779, 0.1096, 0.4257, 0.0342, 0.0249, 1.0], False)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correlation.head()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "0cb24dc9-60a5-4aac-8cec-10f737f5d932",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.10245367, 0.11329165, 0.11153614, 0.06527345,\n",
       "        0.0944388 , 0.12994742],\n",
       "       [0.10245367, 1.        , 0.75824038, 0.91812516, 0.51696382,\n",
       "        0.16550589, 0.17785614],\n",
       "       [0.11329165, 0.75824038, 1.        , 0.6648986 , 0.58552344,\n",
       "        0.60046731, 0.10962705],\n",
       "       [0.11153614, 0.91812516, 0.6648986 , 1.        , 0.42204717,\n",
       "        0.12591661, 0.42567157],\n",
       "       [0.06527345, 0.51696382, 0.58552344, 0.42204717, 1.        ,\n",
       "        0.33942363, 0.03423813],\n",
       "       [0.0944388 , 0.16550589, 0.60046731, 0.12591661, 0.33942363,\n",
       "        1.        , 0.02486967],\n",
       "       [0.12994742, 0.17785614, 0.10962705, 0.42567157, 0.03423813,\n",
       "        0.02486967, 1.        ]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correlation_array = correlation.head()[0].toArray()\n",
    "correlation_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "52da6ce2-924e-4140-b9f9-1607642e910a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "      <th>calories</th>\n",
       "      <th>protein</th>\n",
       "      <th>fat</th>\n",
       "      <th>sodium</th>\n",
       "      <th>protein_ratio</th>\n",
       "      <th>fat_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.102454</td>\n",
       "      <td>0.113292</td>\n",
       "      <td>0.111536</td>\n",
       "      <td>0.065273</td>\n",
       "      <td>0.094439</td>\n",
       "      <td>0.129947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calories</th>\n",
       "      <td>0.102454</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.758240</td>\n",
       "      <td>0.918125</td>\n",
       "      <td>0.516964</td>\n",
       "      <td>0.165506</td>\n",
       "      <td>0.177856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>protein</th>\n",
       "      <td>0.113292</td>\n",
       "      <td>0.758240</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.664899</td>\n",
       "      <td>0.585523</td>\n",
       "      <td>0.600467</td>\n",
       "      <td>0.109627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fat</th>\n",
       "      <td>0.111536</td>\n",
       "      <td>0.918125</td>\n",
       "      <td>0.664899</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.422047</td>\n",
       "      <td>0.125917</td>\n",
       "      <td>0.425672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sodium</th>\n",
       "      <td>0.065273</td>\n",
       "      <td>0.516964</td>\n",
       "      <td>0.585523</td>\n",
       "      <td>0.422047</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.339424</td>\n",
       "      <td>0.034238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>protein_ratio</th>\n",
       "      <td>0.094439</td>\n",
       "      <td>0.165506</td>\n",
       "      <td>0.600467</td>\n",
       "      <td>0.125917</td>\n",
       "      <td>0.339424</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.024870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fat_ratio</th>\n",
       "      <td>0.129947</td>\n",
       "      <td>0.177856</td>\n",
       "      <td>0.109627</td>\n",
       "      <td>0.425672</td>\n",
       "      <td>0.034238</td>\n",
       "      <td>0.024870</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 rating  calories   protein       fat    sodium  \\\n",
       "rating         1.000000  0.102454  0.113292  0.111536  0.065273   \n",
       "calories       0.102454  1.000000  0.758240  0.918125  0.516964   \n",
       "protein        0.113292  0.758240  1.000000  0.664899  0.585523   \n",
       "fat            0.111536  0.918125  0.664899  1.000000  0.422047   \n",
       "sodium         0.065273  0.516964  0.585523  0.422047  1.000000   \n",
       "protein_ratio  0.094439  0.165506  0.600467  0.125917  0.339424   \n",
       "fat_ratio      0.129947  0.177856  0.109627  0.425672  0.034238   \n",
       "\n",
       "               protein_ratio  fat_ratio  \n",
       "rating              0.094439   0.129947  \n",
       "calories            0.165506   0.177856  \n",
       "protein             0.600467   0.109627  \n",
       "fat                 0.125917   0.425672  \n",
       "sodium              0.339424   0.034238  \n",
       "protein_ratio       1.000000   0.024870  \n",
       "fat_ratio           0.024870   1.000000  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correlation_pd = pd.DataFrame(correlation_array,index=CONTINUOUS_COLUMNS,columns=CONTINUOUS_COLUMNS)\n",
    "correlation_pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "21b4d038-b9fb-47ef-8c4c-0956dacf5c7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "      <th>calories</th>\n",
       "      <th>protein</th>\n",
       "      <th>fat</th>\n",
       "      <th>sodium</th>\n",
       "      <th>protein_ratio</th>\n",
       "      <th>fat_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.500</td>\n",
       "      <td>426.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>559.0</td>\n",
       "      <td>0.281690</td>\n",
       "      <td>0.147887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.375</td>\n",
       "      <td>403.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1439.0</td>\n",
       "      <td>0.178660</td>\n",
       "      <td>0.513648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.750</td>\n",
       "      <td>165.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>0.145455</td>\n",
       "      <td>0.381818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.125</td>\n",
       "      <td>547.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>452.0</td>\n",
       "      <td>0.146252</td>\n",
       "      <td>0.526508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.375</td>\n",
       "      <td>948.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>1042.0</td>\n",
       "      <td>0.080169</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15856</th>\n",
       "      <td>3.125</td>\n",
       "      <td>28.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.642857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15857</th>\n",
       "      <td>4.375</td>\n",
       "      <td>671.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>583.0</td>\n",
       "      <td>0.131148</td>\n",
       "      <td>0.375559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15858</th>\n",
       "      <td>4.375</td>\n",
       "      <td>563.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>652.0</td>\n",
       "      <td>0.220249</td>\n",
       "      <td>0.607460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15859</th>\n",
       "      <td>4.375</td>\n",
       "      <td>631.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>517.0</td>\n",
       "      <td>0.285261</td>\n",
       "      <td>0.342314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15860</th>\n",
       "      <td>4.375</td>\n",
       "      <td>560.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3698.0</td>\n",
       "      <td>0.521429</td>\n",
       "      <td>0.160714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15861 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       rating  calories  protein   fat  sodium  protein_ratio  fat_ratio\n",
       "0       2.500     426.0     30.0   7.0   559.0       0.281690   0.147887\n",
       "1       4.375     403.0     18.0  23.0  1439.0       0.178660   0.513648\n",
       "2       3.750     165.0      6.0   7.0   165.0       0.145455   0.381818\n",
       "3       3.125     547.0     20.0  32.0   452.0       0.146252   0.526508\n",
       "4       4.375     948.0     19.0  79.0  1042.0       0.080169   0.750000\n",
       "...       ...       ...      ...   ...     ...            ...        ...\n",
       "15856   3.125      28.0      2.0   2.0    64.0       0.285714   0.642857\n",
       "15857   4.375     671.0     22.0  28.0   583.0       0.131148   0.375559\n",
       "15858   4.375     563.0     31.0  38.0   652.0       0.220249   0.607460\n",
       "15859   4.375     631.0     45.0  24.0   517.0       0.285261   0.342314\n",
       "15860   4.375     560.0     73.0  10.0  3698.0       0.521429   0.160714\n",
       "\n",
       "[15861 rows x 7 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "continuous_features_df = vector_food.toPandas()\n",
    "continuous_features_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "fdeaa859-41ec-45e6-ad0a-820b45656c97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "      <th>calories</th>\n",
       "      <th>protein</th>\n",
       "      <th>fat</th>\n",
       "      <th>sodium</th>\n",
       "      <th>protein_ratio</th>\n",
       "      <th>fat_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.102454</td>\n",
       "      <td>0.113292</td>\n",
       "      <td>0.111536</td>\n",
       "      <td>0.065273</td>\n",
       "      <td>0.094439</td>\n",
       "      <td>0.129947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calories</th>\n",
       "      <td>0.102454</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.758240</td>\n",
       "      <td>0.918125</td>\n",
       "      <td>0.516964</td>\n",
       "      <td>0.165506</td>\n",
       "      <td>0.177856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>protein</th>\n",
       "      <td>0.113292</td>\n",
       "      <td>0.758240</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.664899</td>\n",
       "      <td>0.585523</td>\n",
       "      <td>0.600467</td>\n",
       "      <td>0.109627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fat</th>\n",
       "      <td>0.111536</td>\n",
       "      <td>0.918125</td>\n",
       "      <td>0.664899</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.422047</td>\n",
       "      <td>0.125917</td>\n",
       "      <td>0.425672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sodium</th>\n",
       "      <td>0.065273</td>\n",
       "      <td>0.516964</td>\n",
       "      <td>0.585523</td>\n",
       "      <td>0.422047</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.339424</td>\n",
       "      <td>0.034238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>protein_ratio</th>\n",
       "      <td>0.094439</td>\n",
       "      <td>0.165506</td>\n",
       "      <td>0.600467</td>\n",
       "      <td>0.125917</td>\n",
       "      <td>0.339424</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.024870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fat_ratio</th>\n",
       "      <td>0.129947</td>\n",
       "      <td>0.177856</td>\n",
       "      <td>0.109627</td>\n",
       "      <td>0.425672</td>\n",
       "      <td>0.034238</td>\n",
       "      <td>0.024870</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 rating  calories   protein       fat    sodium  \\\n",
       "rating         1.000000  0.102454  0.113292  0.111536  0.065273   \n",
       "calories       0.102454  1.000000  0.758240  0.918125  0.516964   \n",
       "protein        0.113292  0.758240  1.000000  0.664899  0.585523   \n",
       "fat            0.111536  0.918125  0.664899  1.000000  0.422047   \n",
       "sodium         0.065273  0.516964  0.585523  0.422047  1.000000   \n",
       "protein_ratio  0.094439  0.165506  0.600467  0.125917  0.339424   \n",
       "fat_ratio      0.129947  0.177856  0.109627  0.425672  0.034238   \n",
       "\n",
       "               protein_ratio  fat_ratio  \n",
       "rating              0.094439   0.129947  \n",
       "calories            0.165506   0.177856  \n",
       "protein             0.600467   0.109627  \n",
       "fat                 0.125917   0.425672  \n",
       "sodium              0.339424   0.034238  \n",
       "protein_ratio       1.000000   0.024870  \n",
       "fat_ratio           0.024870   1.000000  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "continuous_features_df.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78c7442f-4e69-4a84-97d1-899a4799e415",
   "metadata": {},
   "source": [
    "作者又在炫技巧了，其实没那么复杂"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "6c0fc1b2-8d5b-4670-adee-7087ba6b964a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20048"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "food.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "205f99d5-7e96-4c35-9d94-867372a83a97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- rating: double (nullable = true)\n",
      " |-- calories: double (nullable = true)\n",
      " |-- protein: double (nullable = true)\n",
      " |-- fat: double (nullable = true)\n",
      " |-- sodium: double (nullable = true)\n",
      " |-- protein_ratio: double (nullable = false)\n",
      " |-- fat_ratio: double (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.select(CONTINUOUS_COLUMNS).printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f420e079-89cd-4b66-a424-b4194c2baf94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+------------------+-----------------+------------------+-----------------+-------------------+-------------------+\n",
      "|summary|           rating|          calories|          protein|               fat|           sodium|      protein_ratio|          fat_ratio|\n",
      "+-------+-----------------+------------------+-----------------+------------------+-----------------+-------------------+-------------------+\n",
      "|  count|            20048|             15932|            15887|             15866|            15930|              20048|              20048|\n",
      "|   mean|3.714460295291301|475.33021591765004|20.13249826902499| 26.82478255388882|576.4514752040176| 0.1171759382402139|0.36674823105015264|\n",
      "| stddev|1.340918766050896|  481.394232922126|29.04177216486837|32.701055376286604| 859.189031083997| 0.1270568607831196|0.28369856795704973|\n",
      "|    min|              0.0|               0.0|              0.0|               0.0|              0.0|                0.0|                0.0|\n",
      "|    25%|             3.75|             198.0|              3.0|               7.0|             80.0|                0.0|                0.0|\n",
      "|    50%|            4.375|             331.0|              8.0|              17.0|            294.0|0.07637231503579953|0.40399002493765584|\n",
      "|    75%|            4.375|             586.0|             27.0|              33.0|            710.0|  0.181360201511335|  0.589041095890411|\n",
      "|    max|              5.0|            3184.0|            173.0|             207.0|           5649.0|  0.861244019138756|              1.125|\n",
      "+-------+-----------------+------------------+-----------------+------------------+-----------------+-------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food.select(CONTINUOUS_COLUMNS).summary().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "a749657a-55cd-4188-bb5f-d1b78b179bcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------------------+\n",
      "|calories|calories_i        |\n",
      "+--------+------------------+\n",
      "|NULL    |475.33021591765004|\n",
      "|NULL    |475.33021591765004|\n",
      "|NULL    |475.33021591765004|\n",
      "|NULL    |475.33021591765004|\n",
      "|NULL    |475.33021591765004|\n",
      "+--------+------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature  import Imputer\n",
    "\n",
    "OLD_COLS = [\"calories\",\"protein\",\"fat\",\"sodium\"]\n",
    "NEW_COLS = [\"calories_i\",\"protein_i\",\"fat_i\",\"sodium_i\"]\n",
    "\n",
    "imputer = Imputer(\n",
    "    strategy = \"mean\",\n",
    "    inputCols = OLD_COLS,\n",
    "    outputCols = NEW_COLS\n",
    ")\n",
    "\n",
    "imputer_model = imputer.fit(food)\n",
    "\n",
    "CONTINUOUS_COLUMNS = (list(set(CONTINUOUS_COLUMNS) - set(OLD_COLS)) + NEW_COLS)\n",
    "\n",
    "food_imputed = imputer_model.transform(food)\n",
    "\n",
    "food_imputed.where(\"calories is null\").select(\"calories\",\"calories_i\").show(5,truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "e7f1a1d3-cdb8-42bf-927b-fe251e11370c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------------------------------------------------------+----------------------------------------------------------------------------------------+\n",
      "|continuous                                                                    |continuous_scaled                                                                       |\n",
      "+------------------------------------------------------------------------------+----------------------------------------------------------------------------------------+\n",
      "|[2.5,426.0,30.0,7.0,559.0]                                                    |[0.5,0.13379396984924624,0.17341040462427745,0.033816425120772944,0.09895556735705434]  |\n",
      "|[4.375,403.0,18.0,23.0,1439.0]                                                |[0.875,0.12657035175879397,0.10404624277456646,0.1111111111111111,0.25473535138962644]  |\n",
      "|[3.75,165.0,6.0,7.0,165.0]                                                    |[0.75,0.05182160804020101,0.03468208092485549,0.033816425120772944,0.029208709506107274]|\n",
      "|[5.0,475.33021591765004,20.13249826902499,26.82478255388882,576.4514752040176]|[1.0,0.14928712811483985,0.1163728223643063,0.12958832151637112,0.10204487080970394]    |\n",
      "|[3.125,547.0,20.0,32.0,452.0]                                                 |[0.625,0.1717964824120603,0.11560693641618497,0.15458937198067632,0.08001416179854841]  |\n",
      "+------------------------------------------------------------------------------+----------------------------------------------------------------------------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import MinMaxScaler\n",
    "\n",
    "CONTINUOUS_NB = [x for x in CONTINUOUS_COLUMNS if \"ratio\" not in x]\n",
    "\n",
    "continuous_assembler = VectorAssembler(inputCols=CONTINUOUS_NB,outputCol=\"continuous\")\n",
    "food_features = continuous_assembler.transform(food_imputed)\n",
    "\n",
    "continuous_scaler = MinMaxScaler(inputCol = \"continuous\",outputCol=\"continuous_scaled\")\n",
    "food_features = continuous_scaler.fit(food_features).transform(food_features)\n",
    "\n",
    "food_features.select( \"continuous\",\"continuous_scaled\").show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "49e3cd8d-d399-4194-9301-b21bc837df7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- title: string (nullable = true)\n",
      " |-- rating: double (nullable = true)\n",
      " |-- calories: double (nullable = true)\n",
      " |-- protein: double (nullable = true)\n",
      " |-- fat: double (nullable = true)\n",
      " |-- sodium: double (nullable = true)\n",
      " |-- cakeweek: double (nullable = true)\n",
      " |-- wasteless: double (nullable = true)\n",
      " |-- 22_minute_meals: double (nullable = true)\n",
      " |-- 3_ingredient_recipes: double (nullable = true)\n",
      " |-- 30_days_of_groceries: double (nullable = true)\n",
      " |-- advance_prep_required: double (nullable = true)\n",
      " |-- alabama: double (nullable = true)\n",
      " |-- alaska: double (nullable = true)\n",
      " |-- alcoholic: double (nullable = true)\n",
      " |-- almond: double (nullable = true)\n",
      " |-- amaretto: double (nullable = true)\n",
      " |-- anchovy: double (nullable = true)\n",
      " |-- anise: double (nullable = true)\n",
      " |-- anniversary: double (nullable = true)\n",
      " |-- anthony_bourdain: double (nullable = true)\n",
      " |-- aperitif: double (nullable = true)\n",
      " |-- appetizer: double (nullable = true)\n",
      " |-- apple: double (nullable = true)\n",
      " |-- apple_juice: double (nullable = true)\n",
      " |-- apricot: double (nullable = true)\n",
      " |-- arizona: double (nullable = true)\n",
      " |-- artichoke: double (nullable = true)\n",
      " |-- arugula: double (nullable = true)\n",
      " |-- asian_pear: double (nullable = true)\n",
      " |-- asparagus: double (nullable = true)\n",
      " |-- aspen: double (nullable = true)\n",
      " |-- atlanta: double (nullable = true)\n",
      " |-- australia: double (nullable = true)\n",
      " |-- avocado: double (nullable = true)\n",
      " |-- back_to_school: double (nullable = true)\n",
      " |-- backyard_bbq: double (nullable = true)\n",
      " |-- bacon: double (nullable = true)\n",
      " |-- bake: double (nullable = true)\n",
      " |-- banana: double (nullable = true)\n",
      " |-- barley: double (nullable = true)\n",
      " |-- basil: double (nullable = true)\n",
      " |-- bass: double (nullable = true)\n",
      " |-- bastille_day: double (nullable = true)\n",
      " |-- bean: double (nullable = true)\n",
      " |-- beef: double (nullable = true)\n",
      " |-- beef_rib: double (nullable = true)\n",
      " |-- beef_shank: double (nullable = true)\n",
      " |-- beef_tenderloin: double (nullable = true)\n",
      " |-- beer: double (nullable = true)\n",
      " |-- beet: double (nullable = true)\n",
      " |-- bell_pepper: double (nullable = true)\n",
      " |-- berry: double (nullable = true)\n",
      " |-- beverly_hills: double (nullable = true)\n",
      " |-- birthday: double (nullable = true)\n",
      " |-- biscuit: double (nullable = true)\n",
      " |-- bitters: double (nullable = true)\n",
      " |-- blackberry: double (nullable = true)\n",
      " |-- blender: double (nullable = true)\n",
      " |-- blue_cheese: double (nullable = true)\n",
      " |-- blueberry: double (nullable = true)\n",
      " |-- boil: double (nullable = true)\n",
      " |-- bok_choy: double (nullable = true)\n",
      " |-- bon_appétit: double (nullable = true)\n",
      " |-- bon_apptit: double (nullable = true)\n",
      " |-- boston: double (nullable = true)\n",
      " |-- bourbon: double (nullable = true)\n",
      " |-- braise: double (nullable = true)\n",
      " |-- bran: double (nullable = true)\n",
      " |-- brandy: double (nullable = true)\n",
      " |-- bread: double (nullable = true)\n",
      " |-- breadcrumbs: double (nullable = true)\n",
      " |-- breakfast: double (nullable = true)\n",
      " |-- brie: double (nullable = true)\n",
      " |-- brine: double (nullable = true)\n",
      " |-- brisket: double (nullable = true)\n",
      " |-- broccoli: double (nullable = true)\n",
      " |-- broccoli_rabe: double (nullable = true)\n",
      " |-- broil: double (nullable = true)\n",
      " |-- brooklyn: double (nullable = true)\n",
      " |-- brown_rice: double (nullable = true)\n",
      " |-- brownie: double (nullable = true)\n",
      " |-- brunch: double (nullable = true)\n",
      " |-- brussel_sprout: double (nullable = true)\n",
      " |-- buffalo: double (nullable = true)\n",
      " |-- buffet: double (nullable = true)\n",
      " |-- bulgaria: double (nullable = true)\n",
      " |-- bulgur: double (nullable = true)\n",
      " |-- burrito: double (nullable = true)\n",
      " |-- butter: double (nullable = true)\n",
      " |-- buttermilk: double (nullable = true)\n",
      " |-- butternut_squash: double (nullable = true)\n",
      " |-- butterscotch_caramel: double (nullable = true)\n",
      " |-- cabbage: double (nullable = true)\n",
      " |-- cake: double (nullable = true)\n",
      " |-- california: double (nullable = true)\n",
      " |-- calvados: double (nullable = true)\n",
      " |-- cambridge: double (nullable = true)\n",
      " |-- campari: double (nullable = true)\n",
      " |-- camping: double (nullable = true)\n",
      " |-- canada: double (nullable = true)\n",
      " |-- candy: double (nullable = true)\n",
      " |-- candy_thermometer: double (nullable = true)\n",
      " |-- cantaloupe: double (nullable = true)\n",
      " |-- capers: double (nullable = true)\n",
      " |-- caraway: double (nullable = true)\n",
      " |-- cardamom: double (nullable = true)\n",
      " |-- carrot: double (nullable = true)\n",
      " |-- cashew: double (nullable = true)\n",
      " |-- casserole_gratin: double (nullable = true)\n",
      " |-- cauliflower: double (nullable = true)\n",
      " |-- caviar: double (nullable = true)\n",
      " |-- celery: double (nullable = true)\n",
      " |-- chambord: double (nullable = true)\n",
      " |-- champagne: double (nullable = true)\n",
      " |-- chard: double (nullable = true)\n",
      " |-- chartreuse: double (nullable = true)\n",
      " |-- cheddar: double (nullable = true)\n",
      " |-- cheese: double (nullable = true)\n",
      " |-- cherry: double (nullable = true)\n",
      " |-- chestnut: double (nullable = true)\n",
      " |-- chicago: double (nullable = true)\n",
      " |-- chicken: double (nullable = true)\n",
      " |-- chickpea: double (nullable = true)\n",
      " |-- chile: double (nullable = true)\n",
      " |-- chile_pepper: double (nullable = true)\n",
      " |-- chili: double (nullable = true)\n",
      " |-- chill: double (nullable = true)\n",
      " |-- chive: double (nullable = true)\n",
      " |-- chocolate: double (nullable = true)\n",
      " |-- christmas: double (nullable = true)\n",
      " |-- christmas_eve: double (nullable = true)\n",
      " |-- cilantro: double (nullable = true)\n",
      " |-- cinco_de_mayo: double (nullable = true)\n",
      " |-- cinnamon: double (nullable = true)\n",
      " |-- citrus: double (nullable = true)\n",
      " |-- clam: double (nullable = true)\n",
      " |-- clove: double (nullable = true)\n",
      " |-- cobbler_crumble: double (nullable = true)\n",
      " |-- cocktail: double (nullable = true)\n",
      " |-- cocktail_party: double (nullable = true)\n",
      " |-- coconut: double (nullable = true)\n",
      " |-- cod: double (nullable = true)\n",
      " |-- coffee: double (nullable = true)\n",
      " |-- coffee_grinder: double (nullable = true)\n",
      " |-- cognac_armagnac: double (nullable = true)\n",
      " |-- collard_greens: double (nullable = true)\n",
      " |-- colorado: double (nullable = true)\n",
      " |-- columbus: double (nullable = true)\n",
      " |-- condiment: double (nullable = true)\n",
      " |-- condiment_spread: double (nullable = true)\n",
      " |-- connecticut: double (nullable = true)\n",
      " |-- cook_like_a_diner: double (nullable = true)\n",
      " |-- cookbook_critic: double (nullable = true)\n",
      " |-- cookie: double (nullable = true)\n",
      " |-- cookies: double (nullable = true)\n",
      " |-- coriander: double (nullable = true)\n",
      " |-- corn: double (nullable = true)\n",
      " |-- cornmeal: double (nullable = true)\n",
      " |-- costa_mesa: double (nullable = true)\n",
      " |-- cottage_cheese: double (nullable = true)\n",
      " |-- couscous: double (nullable = true)\n",
      " |-- crab: double (nullable = true)\n",
      " |-- cranberry: double (nullable = true)\n",
      " |-- cranberry_sauce: double (nullable = true)\n",
      " |-- cream_cheese: double (nullable = true)\n",
      " |-- créme_de_cacao: double (nullable = true)\n",
      " |-- crêpe: double (nullable = true)\n",
      " |-- crme_de_cacao: double (nullable = true)\n",
      " |-- cuba: double (nullable = true)\n",
      " |-- cucumber: double (nullable = true)\n",
      " |-- cumin: double (nullable = true)\n",
      " |-- cupcake: double (nullable = true)\n",
      " |-- currant: double (nullable = true)\n",
      " |-- curry: double (nullable = true)\n",
      " |-- custard: double (nullable = true)\n",
      " |-- dairy: double (nullable = true)\n",
      " |-- dairy_free: double (nullable = true)\n",
      " |-- dallas: double (nullable = true)\n",
      " |-- date: double (nullable = true)\n",
      " |-- deep_fry: double (nullable = true)\n",
      " |-- denver: double (nullable = true)\n",
      " |-- dessert: double (nullable = true)\n",
      " |-- digestif: double (nullable = true)\n",
      " |-- dill: double (nullable = true)\n",
      " |-- dinner: double (nullable = true)\n",
      " |-- dip: double (nullable = true)\n",
      " |-- diwali: double (nullable = true)\n",
      " |-- dominican_republic: double (nullable = true)\n",
      " |-- dorie_greenspan: double (nullable = true)\n",
      " |-- double_boiler: double (nullable = true)\n",
      " |-- dried_fruit: double (nullable = true)\n",
      " |-- drink: double (nullable = true)\n",
      " |-- drinks: double (nullable = true)\n",
      " |-- duck: double (nullable = true)\n",
      " |-- easter: double (nullable = true)\n",
      " |-- eau_de_vie: double (nullable = true)\n",
      " |-- edible_gift: double (nullable = true)\n",
      " |-- egg: double (nullable = true)\n",
      " |-- egg_nog: double (nullable = true)\n",
      " |-- eggplant: double (nullable = true)\n",
      " |-- egypt: double (nullable = true)\n",
      " |-- emeril_lagasse: double (nullable = true)\n",
      " |-- endive: double (nullable = true)\n",
      " |-- engagement_party: double (nullable = true)\n",
      " |-- england: double (nullable = true)\n",
      " |-- entertaining: double (nullable = true)\n",
      " |-- epi__ushg: double (nullable = true)\n",
      " |-- epi_loves_the_microwave: double (nullable = true)\n",
      " |-- escarole: double (nullable = true)\n",
      " |-- fall: double (nullable = true)\n",
      " |-- family_reunion: double (nullable = true)\n",
      " |-- fat_free: double (nullable = true)\n",
      " |-- fathers_day: double (nullable = true)\n",
      " |-- fennel: double (nullable = true)\n",
      " |-- feta: double (nullable = true)\n",
      " |-- fig: double (nullable = true)\n",
      " |-- fish: double (nullable = true)\n",
      " |-- flaming_hot_summer: double (nullable = true)\n",
      " |-- flat_bread: double (nullable = true)\n",
      " |-- florida: double (nullable = true)\n",
      " |-- fontina: double (nullable = true)\n",
      " |-- food_processor: double (nullable = true)\n",
      " |-- fortified_wine: double (nullable = true)\n",
      " |-- fourth_of_july: double (nullable = true)\n",
      " |-- france: double (nullable = true)\n",
      " |-- frangelico: double (nullable = true)\n",
      " |-- frankenrecipe: double (nullable = true)\n",
      " |-- freeze_chill: double (nullable = true)\n",
      " |-- freezer_food: double (nullable = true)\n",
      " |-- friendsgiving: double (nullable = true)\n",
      " |-- frittata: double (nullable = true)\n",
      " |-- fritter: double (nullable = true)\n",
      " |-- frozen_dessert: double (nullable = true)\n",
      " |-- fruit: double (nullable = true)\n",
      " |-- fruit_juice: double (nullable = true)\n",
      " |-- fry: double (nullable = true)\n",
      " |-- game: double (nullable = true)\n",
      " |-- garlic: double (nullable = true)\n",
      " |-- georgia: double (nullable = true)\n",
      " |-- germany: double (nullable = true)\n",
      " |-- gin: double (nullable = true)\n",
      " |-- ginger: double (nullable = true)\n",
      " |-- goat_cheese: double (nullable = true)\n",
      " |-- goose: double (nullable = true)\n",
      " |-- gouda: double (nullable = true)\n",
      " |-- gourmet: double (nullable = true)\n",
      " |-- graduation: double (nullable = true)\n",
      " |-- grains: double (nullable = true)\n",
      " |-- grand_marnier: double (nullable = true)\n",
      " |-- granola: double (nullable = true)\n",
      " |-- grape: double (nullable = true)\n",
      " |-- grapefruit: double (nullable = true)\n",
      " |-- grappa: double (nullable = true)\n",
      " |-- green_bean: double (nullable = true)\n",
      " |-- green_onion_scallion: double (nullable = true)\n",
      " |-- grill: double (nullable = true)\n",
      " |-- grill_barbecue: double (nullable = true)\n",
      " |-- ground_beef: double (nullable = true)\n",
      " |-- ground_lamb: double (nullable = true)\n",
      " |-- guam: double (nullable = true)\n",
      " |-- guava: double (nullable = true)\n",
      " |-- haiti: double (nullable = true)\n",
      " |-- halibut: double (nullable = true)\n",
      " |-- halloween: double (nullable = true)\n",
      " |-- ham: double (nullable = true)\n",
      " |-- hamburger: double (nullable = true)\n",
      " |-- hanukkah: double (nullable = true)\n",
      " |-- harpercollins: double (nullable = true)\n",
      " |-- hawaii: double (nullable = true)\n",
      " |-- hazelnut: double (nullable = true)\n",
      " |-- healdsburg: double (nullable = true)\n",
      " |-- healthy: double (nullable = true)\n",
      " |-- herb: double (nullable = true)\n",
      " |-- high_fiber: double (nullable = true)\n",
      " |-- hollywood: double (nullable = true)\n",
      " |-- hominy_cornmeal_masa: double (nullable = true)\n",
      " |-- honey: double (nullable = true)\n",
      " |-- honeydew: double (nullable = true)\n",
      " |-- hors_doeuvre: double (nullable = true)\n",
      " |-- horseradish: double (nullable = true)\n",
      " |-- hot_drink: double (nullable = true)\n",
      " |-- hot_pepper: double (nullable = true)\n",
      " |-- house_and_garden: double (nullable = true)\n",
      " |-- house_cocktail: double (nullable = true)\n",
      " |-- houston: double (nullable = true)\n",
      " |-- hummus: double (nullable = true)\n",
      " |-- ice_cream: double (nullable = true)\n",
      " |-- ice_cream_machine: double (nullable = true)\n",
      " |-- iced_coffee: double (nullable = true)\n",
      " |-- iced_tea: double (nullable = true)\n",
      " |-- idaho: double (nullable = true)\n",
      " |-- illinois: double (nullable = true)\n",
      " |-- indiana: double (nullable = true)\n",
      " |-- iowa: double (nullable = true)\n",
      " |-- ireland: double (nullable = true)\n",
      " |-- israel: double (nullable = true)\n",
      " |-- italy: double (nullable = true)\n",
      " |-- jalapeño: double (nullable = true)\n",
      " |-- jam_or_jelly: double (nullable = true)\n",
      " |-- jamaica: double (nullable = true)\n",
      " |-- japan: double (nullable = true)\n",
      " |-- jerusalem_artichoke: double (nullable = true)\n",
      " |-- juicer: double (nullable = true)\n",
      " |-- jícama: double (nullable = true)\n",
      " |-- kahlúa: double (nullable = true)\n",
      " |-- kale: double (nullable = true)\n",
      " |-- kansas: double (nullable = true)\n",
      " |-- kansas_city: double (nullable = true)\n",
      " |-- kentucky: double (nullable = true)\n",
      " |-- kentucky_derby: double (nullable = true)\n",
      " |-- kid_friendly: double (nullable = true)\n",
      " |-- kidney_friendly: double (nullable = true)\n",
      " |-- kirsch: double (nullable = true)\n",
      " |-- kitchen_olympics: double (nullable = true)\n",
      " |-- kiwi: double (nullable = true)\n",
      " |-- kosher: double (nullable = true)\n",
      " |-- kosher_for_passover: double (nullable = true)\n",
      " |-- kumquat: double (nullable = true)\n",
      " |-- kwanzaa: double (nullable = true)\n",
      " |-- labor_day: double (nullable = true)\n",
      " |-- lamb: double (nullable = true)\n",
      " |-- lamb_chop: double (nullable = true)\n",
      " |-- lamb_shank: double (nullable = true)\n",
      " |-- lancaster: double (nullable = true)\n",
      " |-- las_vegas: double (nullable = true)\n",
      " |-- lasagna: double (nullable = true)\n",
      " |-- leafy_green: double (nullable = true)\n",
      " |-- leek: double (nullable = true)\n",
      " |-- legume: double (nullable = true)\n",
      " |-- lemon: double (nullable = true)\n",
      " |-- lemon_juice: double (nullable = true)\n",
      " |-- lemongrass: double (nullable = true)\n",
      " |-- lentil: double (nullable = true)\n",
      " |-- lettuce: double (nullable = true)\n",
      " |-- lima_bean: double (nullable = true)\n",
      " |-- lime: double (nullable = true)\n",
      " |-- lime_juice: double (nullable = true)\n",
      " |-- lingonberry: double (nullable = true)\n",
      " |-- liqueur: double (nullable = true)\n",
      " |-- lobster: double (nullable = true)\n",
      " |-- london: double (nullable = true)\n",
      " |-- long_beach: double (nullable = true)\n",
      " |-- los_angeles: double (nullable = true)\n",
      " |-- louisiana: double (nullable = true)\n",
      " |-- louisville: double (nullable = true)\n",
      " |-- low_cal: double (nullable = true)\n",
      " |-- low_carb: double (nullable = true)\n",
      " |-- low_cholesterol: double (nullable = true)\n",
      " |-- low_fat: double (nullable = true)\n",
      " |-- low_sodium: double (nullable = true)\n",
      " |-- low_sugar: double (nullable = true)\n",
      " |-- low_no_sugar: double (nullable = true)\n",
      " |-- lunar_new_year: double (nullable = true)\n",
      " |-- lunch: double (nullable = true)\n",
      " |-- lychee: double (nullable = true)\n",
      " |-- macadamia_nut: double (nullable = true)\n",
      " |-- macaroni_and_cheese: double (nullable = true)\n",
      " |-- maine: double (nullable = true)\n",
      " |-- mandoline: double (nullable = true)\n",
      " |-- mango: double (nullable = true)\n",
      " |-- maple_syrup: double (nullable = true)\n",
      " |-- mardi_gras: double (nullable = true)\n",
      " |-- margarita: double (nullable = true)\n",
      " |-- marinade: double (nullable = true)\n",
      " |-- marinate: double (nullable = true)\n",
      " |-- marsala: double (nullable = true)\n",
      " |-- marscarpone: double (nullable = true)\n",
      " |-- marshmallow: double (nullable = true)\n",
      " |-- martini: double (nullable = true)\n",
      " |-- maryland: double (nullable = true)\n",
      " |-- massachusetts: double (nullable = true)\n",
      " |-- mayonnaise: double (nullable = true)\n",
      " |-- meat: double (nullable = true)\n",
      " |-- meatball: double (nullable = true)\n",
      " |-- meatloaf: double (nullable = true)\n",
      " |-- melon: double (nullable = true)\n",
      " |-- mexico: double (nullable = true)\n",
      " |-- mezcal: double (nullable = true)\n",
      " |-- miami: double (nullable = true)\n",
      " |-- michigan: double (nullable = true)\n",
      " |-- microwave: double (nullable = true)\n",
      " |-- midori: double (nullable = true)\n",
      " |-- milk_cream: double (nullable = true)\n",
      " |-- minneapolis: double (nullable = true)\n",
      " |-- minnesota: double (nullable = true)\n",
      " |-- mint: double (nullable = true)\n",
      " |-- mississippi: double (nullable = true)\n",
      " |-- missouri: double (nullable = true)\n",
      " |-- mixer: double (nullable = true)\n",
      " |-- molasses: double (nullable = true)\n",
      " |-- monterey_jack: double (nullable = true)\n",
      " |-- mortar_and_pestle: double (nullable = true)\n",
      " |-- mothers_day: double (nullable = true)\n",
      " |-- mozzarella: double (nullable = true)\n",
      " |-- muffin: double (nullable = true)\n",
      " |-- mushroom: double (nullable = true)\n",
      " |-- mussel: double (nullable = true)\n",
      " |-- mustard: double (nullable = true)\n",
      " |-- mustard_greens: double (nullable = true)\n",
      " |-- nancy_silverton: double (nullable = true)\n",
      " |-- nebraska: double (nullable = true)\n",
      " |-- nectarine: double (nullable = true)\n",
      " |-- new_hampshire: double (nullable = true)\n",
      " |-- new_jersey: double (nullable = true)\n",
      " |-- new_mexico: double (nullable = true)\n",
      " |-- new_orleans: double (nullable = true)\n",
      " |-- new_years_day: double (nullable = true)\n",
      " |-- new_years_eve: double (nullable = true)\n",
      " |-- new_york: double (nullable = true)\n",
      " |-- no_meat_no_problem: double (nullable = true)\n",
      " |-- no_sugar_added: double (nullable = true)\n",
      " |-- no_cook: double (nullable = true)\n",
      " |-- non_alcoholic: double (nullable = true)\n",
      " |-- noodle: double (nullable = true)\n",
      " |-- north_carolina: double (nullable = true)\n",
      " |-- nut: double (nullable = true)\n",
      " |-- nutmeg: double (nullable = true)\n",
      " |-- oat: double (nullable = true)\n",
      " |-- oatmeal: double (nullable = true)\n",
      " |-- octopus: double (nullable = true)\n",
      " |-- ohio: double (nullable = true)\n",
      " |-- oklahoma: double (nullable = true)\n",
      " |-- okra: double (nullable = true)\n",
      " |-- oktoberfest: double (nullable = true)\n",
      " |-- olive: double (nullable = true)\n",
      " |-- omelet: double (nullable = true)\n",
      " |-- one_pot_meal: double (nullable = true)\n",
      " |-- onion: double (nullable = true)\n",
      " |-- orange: double (nullable = true)\n",
      " |-- orange_juice: double (nullable = true)\n",
      " |-- oregano: double (nullable = true)\n",
      " |-- oregon: double (nullable = true)\n",
      " |-- organic: double (nullable = true)\n",
      " |-- orzo: double (nullable = true)\n",
      " |-- oscars: double (nullable = true)\n",
      " |-- oyster: double (nullable = true)\n",
      " |-- pacific_palisades: double (nullable = true)\n",
      " |-- paleo: double (nullable = true)\n",
      " |-- pan_fry: double (nullable = true)\n",
      " |-- pancake: double (nullable = true)\n",
      " |-- papaya: double (nullable = true)\n",
      " |-- paprika: double (nullable = true)\n",
      " |-- parade: double (nullable = true)\n",
      " |-- paris: double (nullable = true)\n",
      " |-- parmesan: double (nullable = true)\n",
      " |-- parsley: double (nullable = true)\n",
      " |-- parsnip: double (nullable = true)\n",
      " |-- party: double (nullable = true)\n",
      " |-- pasadena: double (nullable = true)\n",
      " |-- passion_fruit: double (nullable = true)\n",
      " |-- passover: double (nullable = true)\n",
      " |-- pasta: double (nullable = true)\n",
      " |-- pasta_maker: double (nullable = true)\n",
      " |-- pastry: double (nullable = true)\n",
      " |-- pea: double (nullable = true)\n",
      " |-- peach: double (nullable = true)\n",
      " |-- peanut: double (nullable = true)\n",
      " |-- peanut_butter: double (nullable = true)\n",
      " |-- peanut_free: double (nullable = true)\n",
      " |-- pear: double (nullable = true)\n",
      " |-- pecan: double (nullable = true)\n",
      " |-- pennsylvania: double (nullable = true)\n",
      " |-- pepper: double (nullable = true)\n",
      " |-- pernod: double (nullable = true)\n",
      " |-- persian_new_year: double (nullable = true)\n",
      " |-- persimmon: double (nullable = true)\n",
      " |-- peru: double (nullable = true)\n",
      " |-- pescatarian: double (nullable = true)\n",
      " |-- philippines: double (nullable = true)\n",
      " |-- phyllo_puff_pastry_dough: double (nullable = true)\n",
      " |-- pickles: double (nullable = true)\n",
      " |-- picnic: double (nullable = true)\n",
      " |-- pie: double (nullable = true)\n",
      " |-- pine_nut: double (nullable = true)\n",
      " |-- pineapple: double (nullable = true)\n",
      " |-- pistachio: double (nullable = true)\n",
      " |-- pittsburgh: double (nullable = true)\n",
      " |-- pizza: double (nullable = true)\n",
      " |-- plantain: double (nullable = true)\n",
      " |-- plum: double (nullable = true)\n",
      " |-- poach: double (nullable = true)\n",
      " |-- poblano: double (nullable = true)\n",
      " |-- poker_game_night: double (nullable = true)\n",
      " |-- pomegranate: double (nullable = true)\n",
      " |-- pomegranate_juice: double (nullable = true)\n",
      " |-- poppy: double (nullable = true)\n",
      " |-- pork: double (nullable = true)\n",
      " |-- pork_chop: double (nullable = true)\n",
      " |-- pork_rib: double (nullable = true)\n",
      " |-- pork_tenderloin: double (nullable = true)\n",
      " |-- port: double (nullable = true)\n",
      " |-- portland: double (nullable = true)\n",
      " |-- pot_pie: double (nullable = true)\n",
      " |-- potato: double (nullable = true)\n",
      " |-- potato_salad: double (nullable = true)\n",
      " |-- potluck: double (nullable = true)\n",
      " |-- poultry: double (nullable = true)\n",
      " |-- poultry_sausage: double (nullable = true)\n",
      " |-- pressure_cooker: double (nullable = true)\n",
      " |-- prosciutto: double (nullable = true)\n",
      " |-- providence: double (nullable = true)\n",
      " |-- prune: double (nullable = true)\n",
      " |-- pumpkin: double (nullable = true)\n",
      " |-- punch: double (nullable = true)\n",
      " |-- purim: double (nullable = true)\n",
      " |-- quail: double (nullable = true)\n",
      " |-- quiche: double (nullable = true)\n",
      " |-- quick_and_easy: double (nullable = true)\n",
      " |-- quick_and_healthy: double (nullable = true)\n",
      " |-- quince: double (nullable = true)\n",
      " |-- quinoa: double (nullable = true)\n",
      " |-- rabbit: double (nullable = true)\n",
      " |-- rack_of_lamb: double (nullable = true)\n",
      " |-- radicchio: double (nullable = true)\n",
      " |-- radish: double (nullable = true)\n",
      " |-- raisin: double (nullable = true)\n",
      " |-- ramadan: double (nullable = true)\n",
      " |-- ramekin: double (nullable = true)\n",
      " |-- raspberry: double (nullable = true)\n",
      " |-- raw: double (nullable = true)\n",
      " |-- red_wine: double (nullable = true)\n",
      " |-- rhode_island: double (nullable = true)\n",
      " |-- rhubarb: double (nullable = true)\n",
      " |-- rice: double (nullable = true)\n",
      " |-- ricotta: double (nullable = true)\n",
      " |-- roast: double (nullable = true)\n",
      " |-- root_vegetable: double (nullable = true)\n",
      " |-- rosemary: double (nullable = true)\n",
      " |-- rosh_hashanah_yom_kippur: double (nullable = true)\n",
      " |-- rosé: double (nullable = true)\n",
      " |-- rub: double (nullable = true)\n",
      " |-- rum: double (nullable = true)\n",
      " |-- rutabaga: double (nullable = true)\n",
      " |-- rye: double (nullable = true)\n",
      " |-- saffron: double (nullable = true)\n",
      " |-- sage: double (nullable = true)\n",
      " |-- sake: double (nullable = true)\n",
      " |-- salad: double (nullable = true)\n",
      " |-- salad_dressing: double (nullable = true)\n",
      " |-- salmon: double (nullable = true)\n",
      " |-- salsa: double (nullable = true)\n",
      " |-- san_francisco: double (nullable = true)\n",
      " |-- sandwich: double (nullable = true)\n",
      " |-- sandwich_theory: double (nullable = true)\n",
      " |-- sangria: double (nullable = true)\n",
      " |-- santa_monica: double (nullable = true)\n",
      " |-- sardine: double (nullable = true)\n",
      " |-- sauce: double (nullable = true)\n",
      " |-- sausage: double (nullable = true)\n",
      " |-- sauté: double (nullable = true)\n",
      " |-- scallop: double (nullable = true)\n",
      " |-- scotch: double (nullable = true)\n",
      " |-- seafood: double (nullable = true)\n",
      " |-- seattle: double (nullable = true)\n",
      " |-- seed: double (nullable = true)\n",
      " |-- self: double (nullable = true)\n",
      " |-- semolina: double (nullable = true)\n",
      " |-- sesame: double (nullable = true)\n",
      " |-- sesame_oil: double (nullable = true)\n",
      " |-- shallot: double (nullable = true)\n",
      " |-- shavuot: double (nullable = true)\n",
      " |-- shellfish: double (nullable = true)\n",
      " |-- sherry: double (nullable = true)\n",
      " |-- shower: double (nullable = true)\n",
      " |-- shrimp: double (nullable = true)\n",
      " |-- side: double (nullable = true)\n",
      " |-- simmer: double (nullable = true)\n",
      " |-- skewer: double (nullable = true)\n",
      " |-- slow_cooker: double (nullable = true)\n",
      " |-- smoker: double (nullable = true)\n",
      " |-- smoothie: double (nullable = true)\n",
      " |-- snapper: double (nullable = true)\n",
      " |-- sorbet: double (nullable = true)\n",
      " |-- soufflé_meringue: double (nullable = true)\n",
      " |-- soup_stew: double (nullable = true)\n",
      " |-- sour_cream: double (nullable = true)\n",
      " |-- sourdough: double (nullable = true)\n",
      " |-- south_carolina: double (nullable = true)\n",
      " |-- soy: double (nullable = true)\n",
      " |-- soy_free: double (nullable = true)\n",
      " |-- soy_sauce: double (nullable = true)\n",
      " |-- spain: double (nullable = true)\n",
      " |-- sparkling_wine: double (nullable = true)\n",
      " |-- spice: double (nullable = true)\n",
      " |-- spinach: double (nullable = true)\n",
      " |-- spirit: double (nullable = true)\n",
      " |-- spring: double (nullable = true)\n",
      " |-- spritzer: double (nullable = true)\n",
      " |-- squash: double (nullable = true)\n",
      " |-- squid: double (nullable = true)\n",
      " |-- st_louis: double (nullable = true)\n",
      " |-- st_patricks_day: double (nullable = true)\n",
      " |-- steak: double (nullable = true)\n",
      " |-- steam: double (nullable = true)\n",
      " |-- stew: double (nullable = true)\n",
      " |-- stir_fry: double (nullable = true)\n",
      " |-- stock: double (nullable = true)\n",
      " |-- strawberry: double (nullable = true)\n",
      " |-- stuffing_dressing: double (nullable = true)\n",
      " |-- sugar_conscious: double (nullable = true)\n",
      " |-- sugar_snap_pea: double (nullable = true)\n",
      " |-- sukkot: double (nullable = true)\n",
      " |-- summer: double (nullable = true)\n",
      " |-- super_bowl: double (nullable = true)\n",
      " |-- suzanne_goin: double (nullable = true)\n",
      " |-- sweet_potato_yam: double (nullable = true)\n",
      " |-- swiss_cheese: double (nullable = true)\n",
      " |-- switzerland: double (nullable = true)\n",
      " |-- swordfish: double (nullable = true)\n",
      " |-- taco: double (nullable = true)\n",
      " |-- tailgating: double (nullable = true)\n",
      " |-- tamarind: double (nullable = true)\n",
      " |-- tangerine: double (nullable = true)\n",
      " |-- tapioca: double (nullable = true)\n",
      " |-- tarragon: double (nullable = true)\n",
      " |-- tart: double (nullable = true)\n",
      " |-- tea: double (nullable = true)\n",
      " |-- tennessee: double (nullable = true)\n",
      " |-- tequila: double (nullable = true)\n",
      " |-- tested_and_improved: double (nullable = true)\n",
      " |-- texas: double (nullable = true)\n",
      " |-- thanksgiving: double (nullable = true)\n",
      " |-- thyme: double (nullable = true)\n",
      " |-- tilapia: double (nullable = true)\n",
      " |-- tofu: double (nullable = true)\n",
      " |-- tomatillo: double (nullable = true)\n",
      " |-- tomato: double (nullable = true)\n",
      " |-- tortillas: double (nullable = true)\n",
      " |-- tree_nut: double (nullable = true)\n",
      " |-- tree_nut_free: double (nullable = true)\n",
      " |-- triple_sec: double (nullable = true)\n",
      " |-- tropical_fruit: double (nullable = true)\n",
      " |-- trout: double (nullable = true)\n",
      " |-- tuna: double (nullable = true)\n",
      " |-- turnip: double (nullable = true)\n",
      " |-- utah: double (nullable = true)\n",
      " |-- valentines_day: double (nullable = true)\n",
      " |-- vanilla: double (nullable = true)\n",
      " |-- veal: double (nullable = true)\n",
      " |-- vegan: double (nullable = true)\n",
      " |-- vegetable: double (nullable = true)\n",
      " |-- vegetarian: double (nullable = true)\n",
      " |-- venison: double (nullable = true)\n",
      " |-- vermont: double (nullable = true)\n",
      " |-- vermouth: double (nullable = true)\n",
      " |-- vinegar: double (nullable = true)\n",
      " |-- virginia: double (nullable = true)\n",
      " |-- vodka: double (nullable = true)\n",
      " |-- waffle: double (nullable = true)\n",
      " |-- walnut: double (nullable = true)\n",
      " |-- wasabi: double (nullable = true)\n",
      " |-- washington: double (nullable = true)\n",
      " |-- washington_dc: double (nullable = true)\n",
      " |-- watercress: double (nullable = true)\n",
      " |-- watermelon: double (nullable = true)\n",
      " |-- wedding: double (nullable = true)\n",
      " |-- weelicious: double (nullable = true)\n",
      " |-- west_virginia: double (nullable = true)\n",
      " |-- westwood: double (nullable = true)\n",
      " |-- wheat_gluten_free: double (nullable = true)\n",
      " |-- whiskey: double (nullable = true)\n",
      " |-- white_wine: double (nullable = true)\n",
      " |-- whole_wheat: double (nullable = true)\n",
      " |-- wild_rice: double (nullable = true)\n",
      " |-- windsor: double (nullable = true)\n",
      " |-- wine: double (nullable = true)\n",
      " |-- winter: double (nullable = true)\n",
      " |-- wisconsin: double (nullable = true)\n",
      " |-- wok: double (nullable = true)\n",
      " |-- yellow_squash: double (nullable = true)\n",
      " |-- yogurt: double (nullable = true)\n",
      " |-- yonkers: double (nullable = true)\n",
      " |-- yuca: double (nullable = true)\n",
      " |-- zucchini: double (nullable = true)\n",
      " |-- cookbooks: double (nullable = true)\n",
      " |-- leftovers: double (nullable = true)\n",
      " |-- snack: double (nullable = true)\n",
      " |-- snack_week: double (nullable = true)\n",
      " |-- turkey: double (nullable = true)\n",
      " |-- protein_ratio: double (nullable = false)\n",
      " |-- fat_ratio: double (nullable = false)\n",
      " |-- calories_i: double (nullable = true)\n",
      " |-- protein_i: double (nullable = true)\n",
      " |-- fat_i: double (nullable = true)\n",
      " |-- sodium_i: double (nullable = true)\n",
      " |-- continuous: vector (nullable = true)\n",
      " |-- continuous_scaled: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "food_features.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "d8e629c7-ab50-4fd2-9c2a-b77897337772",
   "metadata": {},
   "outputs": [],
   "source": [
    "food_features.toPandas().to_csv(\"../data/food_features.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "db017e88-bff6-4c59-8c1c-ce88ebbae356",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['title']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IDENTIFIERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "aa938a6d-6a36-49c5-bf20-2392f70fead9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['protein_ratio',\n",
       " 'rating',\n",
       " 'fat_ratio',\n",
       " 'calories_i',\n",
       " 'protein_i',\n",
       " 'fat_i',\n",
       " 'sodium_i']"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CONTINUOUS_COLUMNS "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "04e4172c-3069-4646-86fd-2baf7d0dbe30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dessert']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TARGET_COLUMN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a2e1cfda-ceb2-46a0-a575-58d8457dff2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'22_minute_meals',\n",
       " '3_ingredient_recipes',\n",
       " 'advance_prep_required',\n",
       " 'alcoholic',\n",
       " 'almond',\n",
       " 'amaretto',\n",
       " 'anchovy',\n",
       " 'anise',\n",
       " 'anniversary',\n",
       " 'aperitif',\n",
       " 'appetizer',\n",
       " 'apple',\n",
       " 'apricot',\n",
       " 'artichoke',\n",
       " 'arugula',\n",
       " 'asian_pear',\n",
       " 'asparagus',\n",
       " 'avocado',\n",
       " 'back_to_school',\n",
       " 'backyard_bbq',\n",
       " 'bacon',\n",
       " 'bake',\n",
       " 'banana',\n",
       " 'barley',\n",
       " 'basil',\n",
       " 'bass',\n",
       " 'bastille_day',\n",
       " 'bean',\n",
       " 'beef',\n",
       " 'beef_rib',\n",
       " 'beef_shank',\n",
       " 'beef_tenderloin',\n",
       " 'beer',\n",
       " 'beet',\n",
       " 'bell_pepper',\n",
       " 'berry',\n",
       " 'birthday',\n",
       " 'bitters',\n",
       " 'blackberry',\n",
       " 'blender',\n",
       " 'blue_cheese',\n",
       " 'blueberry',\n",
       " 'boil',\n",
       " 'bok_choy',\n",
       " 'bon_apptit',\n",
       " 'bon_appétit',\n",
       " 'bourbon',\n",
       " 'braise',\n",
       " 'brandy',\n",
       " 'bread',\n",
       " 'breadcrumbs',\n",
       " 'breakfast',\n",
       " 'brie',\n",
       " 'brine',\n",
       " 'brisket',\n",
       " 'broccoli',\n",
       " 'broccoli_rabe',\n",
       " 'broil',\n",
       " 'brown_rice',\n",
       " 'brunch',\n",
       " 'brussel_sprout',\n",
       " 'buffet',\n",
       " 'bulgur',\n",
       " 'butter',\n",
       " 'buttermilk',\n",
       " 'butternut_squash',\n",
       " 'butterscotch_caramel',\n",
       " 'cabbage',\n",
       " 'cake',\n",
       " 'california',\n",
       " 'calvados',\n",
       " 'campari',\n",
       " 'candy',\n",
       " 'candy_thermometer',\n",
       " 'cantaloupe',\n",
       " 'capers',\n",
       " 'caraway',\n",
       " 'cardamom',\n",
       " 'carrot',\n",
       " 'cashew',\n",
       " 'casserole_gratin',\n",
       " 'cauliflower',\n",
       " 'celery',\n",
       " 'chambord',\n",
       " 'champagne',\n",
       " 'chard',\n",
       " 'chartreuse',\n",
       " 'cheddar',\n",
       " 'cheese',\n",
       " 'cherry',\n",
       " 'chestnut',\n",
       " 'chicken',\n",
       " 'chickpea',\n",
       " 'chile',\n",
       " 'chile_pepper',\n",
       " 'chill',\n",
       " 'chive',\n",
       " 'chocolate',\n",
       " 'christmas',\n",
       " 'christmas_eve',\n",
       " 'cilantro',\n",
       " 'cinco_de_mayo',\n",
       " 'cinnamon',\n",
       " 'citrus',\n",
       " 'clam',\n",
       " 'clove',\n",
       " 'cocktail',\n",
       " 'cocktail_party',\n",
       " 'coconut',\n",
       " 'cod',\n",
       " 'coffee',\n",
       " 'coffee_grinder',\n",
       " 'cognac_armagnac',\n",
       " 'collard_greens',\n",
       " 'colorado',\n",
       " 'condiment',\n",
       " 'condiment_spread',\n",
       " 'connecticut',\n",
       " 'cookie',\n",
       " 'cookies',\n",
       " 'coriander',\n",
       " 'corn',\n",
       " 'cornmeal',\n",
       " 'cottage_cheese',\n",
       " 'couscous',\n",
       " 'crab',\n",
       " 'cranberry',\n",
       " 'cream_cheese',\n",
       " 'créme_de_cacao',\n",
       " 'cucumber',\n",
       " 'cumin',\n",
       " 'currant',\n",
       " 'curry',\n",
       " 'dairy',\n",
       " 'dairy_free',\n",
       " 'date',\n",
       " 'deep_fry',\n",
       " 'dill',\n",
       " 'dinner',\n",
       " 'dip',\n",
       " 'diwali',\n",
       " 'double_boiler',\n",
       " 'dried_fruit',\n",
       " 'drink',\n",
       " 'drinks',\n",
       " 'duck',\n",
       " 'easter',\n",
       " 'edible_gift',\n",
       " 'egg',\n",
       " 'eggplant',\n",
       " 'endive',\n",
       " 'engagement_party',\n",
       " 'escarole',\n",
       " 'fall',\n",
       " 'family_reunion',\n",
       " 'fat_free',\n",
       " 'fathers_day',\n",
       " 'fennel',\n",
       " 'feta',\n",
       " 'fig',\n",
       " 'fish',\n",
       " 'flaming_hot_summer',\n",
       " 'florida',\n",
       " 'fontina',\n",
       " 'food_processor',\n",
       " 'fortified_wine',\n",
       " 'fourth_of_july',\n",
       " 'france',\n",
       " 'frangelico',\n",
       " 'freeze_chill',\n",
       " 'frozen_dessert',\n",
       " 'fruit',\n",
       " 'fruit_juice',\n",
       " 'fry',\n",
       " 'game',\n",
       " 'garlic',\n",
       " 'georgia',\n",
       " 'gin',\n",
       " 'ginger',\n",
       " 'goat_cheese',\n",
       " 'goose',\n",
       " 'gouda',\n",
       " 'gourmet',\n",
       " 'graduation',\n",
       " 'grape',\n",
       " 'grapefruit',\n",
       " 'green_bean',\n",
       " 'green_onion_scallion',\n",
       " 'grill',\n",
       " 'grill_barbecue',\n",
       " 'ground_beef',\n",
       " 'ground_lamb',\n",
       " 'guava',\n",
       " 'halibut',\n",
       " 'halloween',\n",
       " 'ham',\n",
       " 'hanukkah',\n",
       " 'harpercollins',\n",
       " 'hazelnut',\n",
       " 'healthy',\n",
       " 'herb',\n",
       " 'high_fiber',\n",
       " 'hominy_cornmeal_masa',\n",
       " 'honey',\n",
       " 'honeydew',\n",
       " 'hors_doeuvre',\n",
       " 'horseradish',\n",
       " 'hot_drink',\n",
       " 'hot_pepper',\n",
       " 'house_and_garden',\n",
       " 'ice_cream',\n",
       " 'ice_cream_machine',\n",
       " 'iced_tea',\n",
       " 'ireland',\n",
       " 'jalapeño',\n",
       " 'jam_or_jelly',\n",
       " 'jerusalem_artichoke',\n",
       " 'jícama',\n",
       " 'kahlúa',\n",
       " 'kale',\n",
       " 'kentucky_derby',\n",
       " 'kid_friendly',\n",
       " 'kidney_friendly',\n",
       " 'kirsch',\n",
       " 'kiwi',\n",
       " 'kosher',\n",
       " 'kosher_for_passover',\n",
       " 'kumquat',\n",
       " 'kwanzaa',\n",
       " 'lamb',\n",
       " 'lamb_chop',\n",
       " 'lamb_shank',\n",
       " 'leafy_green',\n",
       " 'leek',\n",
       " 'legume',\n",
       " 'lemon',\n",
       " 'lemon_juice',\n",
       " 'lemongrass',\n",
       " 'lentil',\n",
       " 'lettuce',\n",
       " 'lima_bean',\n",
       " 'lime',\n",
       " 'lime_juice',\n",
       " 'liqueur',\n",
       " 'lobster',\n",
       " 'los_angeles',\n",
       " 'low_cal',\n",
       " 'low_carb',\n",
       " 'low_cholesterol',\n",
       " 'low_fat',\n",
       " 'low_no_sugar',\n",
       " 'low_sodium',\n",
       " 'low_sugar',\n",
       " 'lunar_new_year',\n",
       " 'lunch',\n",
       " 'macadamia_nut',\n",
       " 'mandoline',\n",
       " 'mango',\n",
       " 'maple_syrup',\n",
       " 'mardi_gras',\n",
       " 'margarita',\n",
       " 'marinate',\n",
       " 'marsala',\n",
       " 'maryland',\n",
       " 'massachusetts',\n",
       " 'mayonnaise',\n",
       " 'meat',\n",
       " 'melon',\n",
       " 'michigan',\n",
       " 'microwave',\n",
       " 'milk_cream',\n",
       " 'minnesota',\n",
       " 'mint',\n",
       " 'missouri',\n",
       " 'mixer',\n",
       " 'molasses',\n",
       " 'monterey_jack',\n",
       " 'mothers_day',\n",
       " 'mozzarella',\n",
       " 'mushroom',\n",
       " 'mussel',\n",
       " 'mustard',\n",
       " 'mustard_greens',\n",
       " 'nectarine',\n",
       " 'new_jersey',\n",
       " 'new_mexico',\n",
       " 'new_years_day',\n",
       " 'new_years_eve',\n",
       " 'new_york',\n",
       " 'no_cook',\n",
       " 'no_sugar_added',\n",
       " 'non_alcoholic',\n",
       " 'noodle',\n",
       " 'north_carolina',\n",
       " 'nut',\n",
       " 'nutmeg',\n",
       " 'oat',\n",
       " 'oatmeal',\n",
       " 'ohio',\n",
       " 'okra',\n",
       " 'oktoberfest',\n",
       " 'olive',\n",
       " 'one_pot_meal',\n",
       " 'onion',\n",
       " 'orange',\n",
       " 'orange_juice',\n",
       " 'oregano',\n",
       " 'oregon',\n",
       " 'oscars',\n",
       " 'oyster',\n",
       " 'paleo',\n",
       " 'pan_fry',\n",
       " 'papaya',\n",
       " 'paprika',\n",
       " 'parade',\n",
       " 'parmesan',\n",
       " 'parsley',\n",
       " 'parsnip',\n",
       " 'party',\n",
       " 'pasadena',\n",
       " 'passion_fruit',\n",
       " 'passover',\n",
       " 'pasta',\n",
       " 'pasta_maker',\n",
       " 'pastry',\n",
       " 'pea',\n",
       " 'peach',\n",
       " 'peanut',\n",
       " 'peanut_butter',\n",
       " 'peanut_free',\n",
       " 'pear',\n",
       " 'pecan',\n",
       " 'pennsylvania',\n",
       " 'pepper',\n",
       " 'pernod',\n",
       " 'persimmon',\n",
       " 'pescatarian',\n",
       " 'phyllo_puff_pastry_dough',\n",
       " 'pickles',\n",
       " 'picnic',\n",
       " 'pie',\n",
       " 'pine_nut',\n",
       " 'pineapple',\n",
       " 'pistachio',\n",
       " 'pittsburgh',\n",
       " 'pizza',\n",
       " 'plantain',\n",
       " 'plum',\n",
       " 'poach',\n",
       " 'poker_game_night',\n",
       " 'pomegranate',\n",
       " 'poppy',\n",
       " 'pork',\n",
       " 'pork_chop',\n",
       " 'pork_rib',\n",
       " 'pork_tenderloin',\n",
       " 'port',\n",
       " 'portland',\n",
       " 'potato',\n",
       " 'potluck',\n",
       " 'poultry',\n",
       " 'poultry_sausage',\n",
       " 'prosciutto',\n",
       " 'prune',\n",
       " 'pumpkin',\n",
       " 'punch',\n",
       " 'purim',\n",
       " 'quail',\n",
       " 'quick_and_easy',\n",
       " 'quick_and_healthy',\n",
       " 'quince',\n",
       " 'quinoa',\n",
       " 'rack_of_lamb',\n",
       " 'radicchio',\n",
       " 'radish',\n",
       " 'raisin',\n",
       " 'ramadan',\n",
       " 'ramekin',\n",
       " 'raspberry',\n",
       " 'raw',\n",
       " 'red_wine',\n",
       " 'rhubarb',\n",
       " 'rice',\n",
       " 'ricotta',\n",
       " 'roast',\n",
       " 'root_vegetable',\n",
       " 'rosemary',\n",
       " 'rosh_hashanah_yom_kippur',\n",
       " 'rum',\n",
       " 'rutabaga',\n",
       " 'rye',\n",
       " 'saffron',\n",
       " 'sage',\n",
       " 'sake',\n",
       " 'salad',\n",
       " 'salad_dressing',\n",
       " 'salmon',\n",
       " 'sandwich',\n",
       " 'sandwich_theory',\n",
       " 'sangria',\n",
       " 'santa_monica',\n",
       " 'sauce',\n",
       " 'sausage',\n",
       " 'sauté',\n",
       " 'scallop',\n",
       " 'scotch',\n",
       " 'seafood',\n",
       " 'seattle',\n",
       " 'seed',\n",
       " 'self',\n",
       " 'semolina',\n",
       " 'sesame',\n",
       " 'sesame_oil',\n",
       " 'shallot',\n",
       " 'shavuot',\n",
       " 'shellfish',\n",
       " 'sherry',\n",
       " 'shower',\n",
       " 'shrimp',\n",
       " 'side',\n",
       " 'simmer',\n",
       " 'slow_cooker',\n",
       " 'smoothie',\n",
       " 'snack',\n",
       " 'snack_week',\n",
       " 'snapper',\n",
       " 'soup_stew',\n",
       " 'sour_cream',\n",
       " 'south_carolina',\n",
       " 'soy',\n",
       " 'soy_free',\n",
       " 'soy_sauce',\n",
       " 'sparkling_wine',\n",
       " 'spice',\n",
       " 'spinach',\n",
       " 'spirit',\n",
       " 'spring',\n",
       " 'squash',\n",
       " 'squid',\n",
       " 'st_patricks_day',\n",
       " 'steak',\n",
       " 'steam',\n",
       " 'stew',\n",
       " 'stir_fry',\n",
       " 'strawberry',\n",
       " 'stuffing_dressing',\n",
       " 'sugar_conscious',\n",
       " 'sugar_snap_pea',\n",
       " 'sukkot',\n",
       " 'summer',\n",
       " 'super_bowl',\n",
       " 'sweet_potato_yam',\n",
       " 'swiss_cheese',\n",
       " 'swordfish',\n",
       " 'tailgating',\n",
       " 'tamarind',\n",
       " 'tangerine',\n",
       " 'tapioca',\n",
       " 'tarragon',\n",
       " 'tea',\n",
       " 'tequila',\n",
       " 'texas',\n",
       " 'thanksgiving',\n",
       " 'thyme',\n",
       " 'tilapia',\n",
       " 'tofu',\n",
       " 'tomatillo',\n",
       " 'tomato',\n",
       " 'tortillas',\n",
       " 'tree_nut',\n",
       " 'tree_nut_free',\n",
       " 'triple_sec',\n",
       " 'tropical_fruit',\n",
       " 'trout',\n",
       " 'tuna',\n",
       " 'turkey',\n",
       " 'turnip',\n",
       " 'valentines_day',\n",
       " 'vanilla',\n",
       " 'veal',\n",
       " 'vegan',\n",
       " 'vegetable',\n",
       " 'vegetarian',\n",
       " 'vermouth',\n",
       " 'vinegar',\n",
       " 'virginia',\n",
       " 'vodka',\n",
       " 'walnut',\n",
       " 'wasabi',\n",
       " 'washington',\n",
       " 'washington_dc',\n",
       " 'watercress',\n",
       " 'watermelon',\n",
       " 'wedding',\n",
       " 'weelicious',\n",
       " 'wheat_gluten_free',\n",
       " 'whiskey',\n",
       " 'white_wine',\n",
       " 'whole_wheat',\n",
       " 'wild_rice',\n",
       " 'wine',\n",
       " 'winter',\n",
       " 'wok',\n",
       " 'yellow_squash',\n",
       " 'yogurt',\n",
       " 'zucchini'}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BINARY_COLUMNS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "21fbf425-f5c2-4e12-8583-e2deecdba257",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "BINARY_COLUMNS = list(BINARY_COLUMNS)\n",
    "with open('../data/binary_cols.json','w') as f:\n",
    "    json.dump(BINARY_COLUMNS,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed1e11c6-b4a6-49a2-b408-7b623377963f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
